Run each test assembly in a test host process of its own

Motus.sln

@@ -57,6 +57,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4.Tests", "tests\Motus.Testing.MSTest.v4.Tests\Motus.Testing.MSTest.v4.Tests.csproj", "{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.WebApp", "tests\Fixtures\Motus.Fixtures.WebApp\Motus.Fixtures.WebApp.csproj", "{12681CE2-69A2-4EC8-A551-C5D1467C2272}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.WebApp.Tests", "tests\Fixtures\Motus.Fixtures.WebApp.Tests\Motus.Fixtures.WebApp.Tests.csproj", "{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.OldMotus.Tests", "tests\Fixtures\Motus.Fixtures.OldMotus.Tests\Motus.Fixtures.OldMotus.Tests.csproj", "{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -355,6 +361,42 @@ Global
 		{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x64.Build.0 = Release|Any CPU
 		{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.ActiveCfg = Release|Any CPU
 		{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.Build.0 = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x64.Build.0 = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x86.Build.0 = Debug|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|Any CPU.Build.0 = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x64.ActiveCfg = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x64.Build.0 = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x86.ActiveCfg = Release|Any CPU
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x86.Build.0 = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x64.Build.0 = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x86.Build.0 = Debug|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x64.ActiveCfg = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x64.Build.0 = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x86.ActiveCfg = Release|Any CPU
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x86.Build.0 = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x64.Build.0 = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x86.Build.0 = Debug|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x64.ActiveCfg = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x64.Build.0 = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x86.ActiveCfg = Release|Any CPU
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -384,6 +426,9 @@ Global
 		{C65BD8A2-DF4C-49C2-A070-8998E55D1C38} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E}
 		{14C675DB-0985-4110-8DBE-585D5AD24571} = {C2BE1865-1543-4EBB-9E95-92EE083BF967}
 		{88F3ADB4-C36D-4DDD-8976-0B4645EE0381} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E}
+		{12681CE2-69A2-4EC8-A551-C5D1467C2272} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E}
+		{0A715F83-AD63-4AE5-B4ED-14FBDB84E231} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E}
+		{9ED082D7-8B0C-464D-A9F1-A17E3BE73038} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3A99AF87-BFA9-49CF-A20A-71753BE54A53}

docs/guides/configuration.md

@@ -16,6 +16,8 @@ Each layer is additive. A value set in a higher-precedence layer is never overwr
 
 When Motus initializes, it walks up from `Environment.CurrentDirectory`, checking each directory for a `motus.config.json` file. The first file found is used. If no file is found anywhere in the directory tree, Motus proceeds with defaults. Discovery is performed once at startup and the result is cached for the lifetime of the process.
 
+Under `motus run`, the tests run in a test host process whose current directory is the test assembly's output folder. The test host looks for `motus.config.json` from the directory `motus run` was started in, as the runner does, so both read the same file. Relative paths in the file are resolved later, when they are used, against the output folder. See [`motus run`](../reference/cli.md#motus-run).
+
 ### Complete Schema
 
 ```json

docs/guides/sharding.md

@@ -25,7 +25,7 @@ A malformed spec fails immediately with a message naming the expected form rathe
 
 ## How tests are assigned
 
-Discovered tests are sorted by a stable key, the assembly path followed by the fully qualified test name, compared ordinally. Reflection does not guarantee a consistent enumeration order across machines or runtime versions, so sorting first is what makes the partition reproducible rather than merely deterministic on one box.
+Discovered tests are sorted by a stable key, the assembly path followed by the test's identity (its class, its method with the parameter types, and for a data row, which data source and which row of it), compared ordinally. The identity rather than the display name is used because a data source can name its rows differently in each process, for example when a row carries a Guid or a time. Reflection does not guarantee a consistent enumeration order across machines or runtime versions, so sorting first is what makes the partition reproducible rather than merely deterministic on one box.
 
 The sorted list is then dealt out round-robin: position `i` belongs to shard `i % total`. Round-robin over a stable sort matters more than it looks. Tests from one class sort together, and a class that is slow is usually slow throughout, so handing out contiguous blocks would drop an entire slow class onto one agent. Dealing alternately spreads those tests across every shard without needing any timing data to do it.
 

docs/guides/testing-frameworks.md

@@ -414,7 +414,7 @@ Or set environment variables:
 | `MOTUS_FAILURES_TRACE` | `true` / `false` / `1` / `0` | Enables or disables trace capture on failure. |
 | `MOTUS_FAILURES_TRACE_PATH` | string | Directory where trace zips are written. |
 
-When `tracePath` / `MOTUS_FAILURES_TRACE_PATH` is not set, traces are written to `test-results/traces` by default. The directory is created automatically if it does not exist.
+When `tracePath` / `MOTUS_FAILURES_TRACE_PATH` is not set, traces are written to `test-results/traces` by default. The directory is created automatically if it does not exist. A relative path resolves against the current directory, which is the test project's output folder under both `dotnet test` and `motus run`; give an absolute path to collect traces somewhere else.
 
 ### Artifact names
 

docs/reference/cli.md

@@ -63,14 +63,23 @@ motus run bin/Release/net8.0/MyTests.dll --shard 1/4 --retries 2
 
 The output path for a reporter is part of its value, so it is `--reporter junit:results.xml` rather than a separate flag. Passing `--reporter junit` with no path writes nowhere.
 
-**How tests run.** `motus run` and the visual runner load the test assembly and call its tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on:
+**Where tests run.** Each test assembly runs in a test host process of its own, the way `dotnet test` runs it. The process is started with the assembly's own `.deps.json` and `.runtimeconfig.json`, so the tests get the versions of their dependencies that they were built with (an app that uses SkiaSharp 3 gets SkiaSharp 3, whatever `motus` itself uses), and the ASP.NET Core and other frameworks they reference. Both `AppContext.BaseDirectory` and the current directory are the assembly's output folder, which is what `WebApplicationFactory` and anything else that reads files beside the tests expects:
+
+- A relative path a test uses, including a relative `failure.tracePath` or `failure.screenshotPath`, resolves against the output folder, as it does under `dotnet test`.
+- `motus.config.json` is still looked up from the directory `motus run` was started in, so the runner and its test hosts read the same settings.
+- The tests run on the Motus version their project references. When that differs from the version of `motus`, `motus run` prints a warning; keep the two on the same version. With an older Motus, anything that version cannot provide, such as accessibility reports, performance metrics or coverage, is left out with a warning, and the tests still run and report their results.
+- A suite built for an older .NET than `motus` runs on the newest .NET installed.
+- A test that ends its process, or crashes it, takes only its test host down: that test fails with the host's exit code, each test of that assembly still to run fails saying it did not run, and other assemblies carry on. Whatever the process wrote before it ended, such as a stack overflow report, is on the console.
+- Starting a test host adds a fraction of a second per assembly. The visual runner starts a new one for each run, so a rebuilt assembly is picked up by the next run without restarting the runner. It finds each test again by its class, method and, for a data row, its place in its data source, so a row whose name changes on every read (one that carries a Guid or a time) still runs; a row its data source no longer returns fails saying so.
+
+**How tests run.** Inside the test host, the runners call the tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on:
 
 - MSTest tests get a `TestContext`, through their `TestContext` property or a constructor that takes one, and `[AssemblyInitialize]`, `[ClassInitialize]` and the matching cleanup methods receive one when they take it. It carries `TestName`, `FullyQualifiedTestClassName` and a `CancellationTokenSource`, and `CurrentTestOutcome` is the test's own outcome by the time `[TestCleanup]` runs. `WriteLine` goes to the console. Files passed to `AddResultFile` are left where they are rather than attached to a result.
 - `[DataRow]`, `[DynamicData]` and any other MSTest data source run as one test per row, named the way MSTest names them, such as `SignInTests.SignsIn (owner)`. `--filter` matches that name, so it can select a single row.
 - Setup methods (`[TestInitialize]`, NUnit `[SetUp]`) run base class first. Cleanup methods (`[TestCleanup]`, NUnit `[TearDown]`) run derived class first and base class last, and each one runs even when an earlier one throws.
 - Attributes derived from `[TestClass]` or `[TestMethod]`, such as `[DataTestMethod]`, `[MotusTestClass]` and `[MotusTestMethod]`, mark tests as the attributes they derive from do. NUnit `[TestCase]` and xUnit `[Theory]` tests are not run.
 
-**Exit code.** `0` when everything passed. `1` when any test failed, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky.
+**Exit code.** `0` when everything passed. `1` when any test failed, when a test assembly could not be loaded or its test host could not start, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky. The tests of the other assemblies still run.
 
 See [Sharding](../guides/sharding.md), [Flaky Tests and Quarantine](../guides/flaky-tests-and-quarantine.md), and [Configuration](../guides/configuration.md) for the settings behind these.
 

src/Motus.Cli/Commands/RunCommand.cs

@@ -191,8 +191,10 @@ public static Command Build()
                 }
             }
 
-            var discovery = new TestDiscovery();
-            var tests = discovery.Discover(assemblies, filter);
+            // Each assembly gets a test host process of its own, which discovery starts and which
+            // runs its tests; disposing the discovery shuts them down.
+            await using var discovery = new TestDiscovery();
+            var tests = await discovery.DiscoverAsync(assemblies, filter, ct);
 
             if (quarantinePath is not null)
             {
@@ -226,8 +228,17 @@ public static Command Build()
                 tests, reporter, a11yMode, perfBudget, coverageReporters, retries, retryPolicy,
                 shardIndex, shardTotal);
 
+            // An assembly named on the command line whose tests could not even be listed fails the run,
+            // so that a test host that cannot start never looks like a green run.
+            if (discovery.FailedAssemblies.Count > 0)
+            {
+                Console.Error.WriteLine(
+                    $"Error: {discovery.FailedAssemblies.Count} test assembly(ies) could not be loaded: {string.Join(", ", discovery.FailedAssemblies)}");
+            }
+
             return (runResult.Failed > 0
                 || runResult.CoverageThresholdsFailed
+                || discovery.FailedAssemblies.Count > 0
                 || (failOnFlaky && runResult.Flaky > 0)) ? 1 : 0;
         });
 
@@ -256,7 +267,7 @@ private static bool TryParseRetryPolicy(string spec, out RetryPolicy policy)
     /// </summary>
     internal static bool IsListed(DiscoveredTest test, IReadOnlySet<string> quarantineList)
         => quarantineList.Contains(test.FullName)
-           || quarantineList.Contains($"{test.TestClass.FullName}.{test.TestMethod.Name}");
+           || quarantineList.Contains($"{test.ClassName}.{test.MethodName}");
 
     private static HashSet<string> LoadQuarantineList(string path)
     {

src/Motus.Cli/Services/ShardSelector.cs

@@ -11,7 +11,8 @@ public static class ShardSelector
     /// <summary>
     /// Returns the subset of <paramref name="tests"/> belonging to shard <paramref name="index"/>
     /// of <paramref name="total"/>. The full set is first sorted by a stable key
-    /// (assembly path + fully qualified name, ordinal) so the order is identical across machines
+    /// (assembly path + test id, ordinal; the id rather than the name, since a data source can name
+    /// its rows differently in each process) so the order is identical across machines
     /// regardless of reflection enumeration order, then assigned round-robin by
     /// <c>position % total == index - 1</c>. Round-robin over a stable sort spreads slow classes
     /// across shards without needing timing data.
@@ -76,5 +77,5 @@ public static bool TryParse(string spec, out int index, out int total, out strin
     }
 
     private static string SortKey(DiscoveredTest test) =>
-        $"{test.TestClass.Assembly.Location}::{test.FullName}";
+        $"{test.AssemblyPath}::{test.TestId}";
 }