diff --git a/Motus.sln b/Motus.sln index c3fbc62..c8ffec2 100644 --- a/Motus.sln +++ b/Motus.sln @@ -57,6 +57,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4.Tests", "tests\Motus.Testing.MSTest.v4.Tests\Motus.Testing.MSTest.v4.Tests.csproj", "{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.WebApp", "tests\Fixtures\Motus.Fixtures.WebApp\Motus.Fixtures.WebApp.csproj", "{12681CE2-69A2-4EC8-A551-C5D1467C2272}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.WebApp.Tests", "tests\Fixtures\Motus.Fixtures.WebApp.Tests\Motus.Fixtures.WebApp.Tests.csproj", "{0A715F83-AD63-4AE5-B4ED-14FBDB84E231}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Fixtures.OldMotus.Tests", "tests\Fixtures\Motus.Fixtures.OldMotus.Tests\Motus.Fixtures.OldMotus.Tests.csproj", "{9ED082D7-8B0C-464D-A9F1-A17E3BE73038}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -355,6 +361,42 @@ Global {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x64.Build.0 = Release|Any CPU {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.ActiveCfg = Release|Any CPU {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.Build.0 = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|Any CPU.Build.0 = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x64.ActiveCfg = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x64.Build.0 = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x86.ActiveCfg = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Debug|x86.Build.0 = Debug|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|Any CPU.ActiveCfg = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|Any CPU.Build.0 = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x64.ActiveCfg = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x64.Build.0 = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x86.ActiveCfg = Release|Any CPU + {12681CE2-69A2-4EC8-A551-C5D1467C2272}.Release|x86.Build.0 = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x64.ActiveCfg = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x64.Build.0 = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x86.ActiveCfg = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Debug|x86.Build.0 = Debug|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|Any CPU.Build.0 = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x64.ActiveCfg = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x64.Build.0 = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x86.ActiveCfg = Release|Any CPU + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231}.Release|x86.Build.0 = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x64.ActiveCfg = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x64.Build.0 = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x86.ActiveCfg = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Debug|x86.Build.0 = Debug|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|Any CPU.Build.0 = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x64.ActiveCfg = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x64.Build.0 = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x86.ActiveCfg = Release|Any CPU + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -384,6 +426,9 @@ Global {C65BD8A2-DF4C-49C2-A070-8998E55D1C38} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} {14C675DB-0985-4110-8DBE-585D5AD24571} = {C2BE1865-1543-4EBB-9E95-92EE083BF967} {88F3ADB4-C36D-4DDD-8976-0B4645EE0381} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} + {12681CE2-69A2-4EC8-A551-C5D1467C2272} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} + {0A715F83-AD63-4AE5-B4ED-14FBDB84E231} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} + {9ED082D7-8B0C-464D-A9F1-A17E3BE73038} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3A99AF87-BFA9-49CF-A20A-71753BE54A53} diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 4afaec8..502240f 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -16,6 +16,8 @@ Each layer is additive. A value set in a higher-precedence layer is never overwr When Motus initializes, it walks up from `Environment.CurrentDirectory`, checking each directory for a `motus.config.json` file. The first file found is used. If no file is found anywhere in the directory tree, Motus proceeds with defaults. Discovery is performed once at startup and the result is cached for the lifetime of the process. +Under `motus run`, the tests run in a test host process whose current directory is the test assembly's output folder. The test host looks for `motus.config.json` from the directory `motus run` was started in, as the runner does, so both read the same file. Relative paths in the file are resolved later, when they are used, against the output folder. See [`motus run`](../reference/cli.md#motus-run). + ### Complete Schema ```json diff --git a/docs/guides/sharding.md b/docs/guides/sharding.md index 66ea4d2..d3fae0d 100644 --- a/docs/guides/sharding.md +++ b/docs/guides/sharding.md @@ -25,7 +25,7 @@ A malformed spec fails immediately with a message naming the expected form rathe ## How tests are assigned -Discovered tests are sorted by a stable key, the assembly path followed by the fully qualified test name, compared ordinally. Reflection does not guarantee a consistent enumeration order across machines or runtime versions, so sorting first is what makes the partition reproducible rather than merely deterministic on one box. +Discovered tests are sorted by a stable key, the assembly path followed by the test's identity (its class, its method with the parameter types, and for a data row, which data source and which row of it), compared ordinally. The identity rather than the display name is used because a data source can name its rows differently in each process, for example when a row carries a Guid or a time. Reflection does not guarantee a consistent enumeration order across machines or runtime versions, so sorting first is what makes the partition reproducible rather than merely deterministic on one box. The sorted list is then dealt out round-robin: position `i` belongs to shard `i % total`. Round-robin over a stable sort matters more than it looks. Tests from one class sort together, and a class that is slow is usually slow throughout, so handing out contiguous blocks would drop an entire slow class onto one agent. Dealing alternately spreads those tests across every shard without needing any timing data to do it. diff --git a/docs/guides/testing-frameworks.md b/docs/guides/testing-frameworks.md index c69f0f5..467849a 100644 --- a/docs/guides/testing-frameworks.md +++ b/docs/guides/testing-frameworks.md @@ -414,7 +414,7 @@ Or set environment variables: | `MOTUS_FAILURES_TRACE` | `true` / `false` / `1` / `0` | Enables or disables trace capture on failure. | | `MOTUS_FAILURES_TRACE_PATH` | string | Directory where trace zips are written. | -When `tracePath` / `MOTUS_FAILURES_TRACE_PATH` is not set, traces are written to `test-results/traces` by default. The directory is created automatically if it does not exist. +When `tracePath` / `MOTUS_FAILURES_TRACE_PATH` is not set, traces are written to `test-results/traces` by default. The directory is created automatically if it does not exist. A relative path resolves against the current directory, which is the test project's output folder under both `dotnet test` and `motus run`; give an absolute path to collect traces somewhere else. ### Artifact names diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ef90e01..2780d09 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -63,14 +63,23 @@ motus run bin/Release/net8.0/MyTests.dll --shard 1/4 --retries 2 The output path for a reporter is part of its value, so it is `--reporter junit:results.xml` rather than a separate flag. Passing `--reporter junit` with no path writes nowhere. -**How tests run.** `motus run` and the visual runner load the test assembly and call its tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on: +**Where tests run.** Each test assembly runs in a test host process of its own, the way `dotnet test` runs it. The process is started with the assembly's own `.deps.json` and `.runtimeconfig.json`, so the tests get the versions of their dependencies that they were built with (an app that uses SkiaSharp 3 gets SkiaSharp 3, whatever `motus` itself uses), and the ASP.NET Core and other frameworks they reference. Both `AppContext.BaseDirectory` and the current directory are the assembly's output folder, which is what `WebApplicationFactory` and anything else that reads files beside the tests expects: + +- A relative path a test uses, including a relative `failure.tracePath` or `failure.screenshotPath`, resolves against the output folder, as it does under `dotnet test`. +- `motus.config.json` is still looked up from the directory `motus run` was started in, so the runner and its test hosts read the same settings. +- The tests run on the Motus version their project references. When that differs from the version of `motus`, `motus run` prints a warning; keep the two on the same version. With an older Motus, anything that version cannot provide, such as accessibility reports, performance metrics or coverage, is left out with a warning, and the tests still run and report their results. +- A suite built for an older .NET than `motus` runs on the newest .NET installed. +- A test that ends its process, or crashes it, takes only its test host down: that test fails with the host's exit code, each test of that assembly still to run fails saying it did not run, and other assemblies carry on. Whatever the process wrote before it ended, such as a stack overflow report, is on the console. +- Starting a test host adds a fraction of a second per assembly. The visual runner starts a new one for each run, so a rebuilt assembly is picked up by the next run without restarting the runner. It finds each test again by its class, method and, for a data row, its place in its data source, so a row whose name changes on every read (one that carries a Guid or a time) still runs; a row its data source no longer returns fails saying so. + +**How tests run.** Inside the test host, the runners call the tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on: - MSTest tests get a `TestContext`, through their `TestContext` property or a constructor that takes one, and `[AssemblyInitialize]`, `[ClassInitialize]` and the matching cleanup methods receive one when they take it. It carries `TestName`, `FullyQualifiedTestClassName` and a `CancellationTokenSource`, and `CurrentTestOutcome` is the test's own outcome by the time `[TestCleanup]` runs. `WriteLine` goes to the console. Files passed to `AddResultFile` are left where they are rather than attached to a result. - `[DataRow]`, `[DynamicData]` and any other MSTest data source run as one test per row, named the way MSTest names them, such as `SignInTests.SignsIn (owner)`. `--filter` matches that name, so it can select a single row. - Setup methods (`[TestInitialize]`, NUnit `[SetUp]`) run base class first. Cleanup methods (`[TestCleanup]`, NUnit `[TearDown]`) run derived class first and base class last, and each one runs even when an earlier one throws. - Attributes derived from `[TestClass]` or `[TestMethod]`, such as `[DataTestMethod]`, `[MotusTestClass]` and `[MotusTestMethod]`, mark tests as the attributes they derive from do. NUnit `[TestCase]` and xUnit `[Theory]` tests are not run. -**Exit code.** `0` when everything passed. `1` when any test failed, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky. +**Exit code.** `0` when everything passed. `1` when any test failed, when a test assembly could not be loaded or its test host could not start, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky. The tests of the other assemblies still run. See [Sharding](../guides/sharding.md), [Flaky Tests and Quarantine](../guides/flaky-tests-and-quarantine.md), and [Configuration](../guides/configuration.md) for the settings behind these. diff --git a/src/Motus.Cli/Commands/RunCommand.cs b/src/Motus.Cli/Commands/RunCommand.cs index 3c82f9e..7085731 100644 --- a/src/Motus.Cli/Commands/RunCommand.cs +++ b/src/Motus.Cli/Commands/RunCommand.cs @@ -191,8 +191,10 @@ public static Command Build() } } - var discovery = new TestDiscovery(); - var tests = discovery.Discover(assemblies, filter); + // Each assembly gets a test host process of its own, which discovery starts and which + // runs its tests; disposing the discovery shuts them down. + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync(assemblies, filter, ct); if (quarantinePath is not null) { @@ -226,8 +228,17 @@ public static Command Build() tests, reporter, a11yMode, perfBudget, coverageReporters, retries, retryPolicy, shardIndex, shardTotal); + // An assembly named on the command line whose tests could not even be listed fails the run, + // so that a test host that cannot start never looks like a green run. + if (discovery.FailedAssemblies.Count > 0) + { + Console.Error.WriteLine( + $"Error: {discovery.FailedAssemblies.Count} test assembly(ies) could not be loaded: {string.Join(", ", discovery.FailedAssemblies)}"); + } + return (runResult.Failed > 0 || runResult.CoverageThresholdsFailed + || discovery.FailedAssemblies.Count > 0 || (failOnFlaky && runResult.Flaky > 0)) ? 1 : 0; }); @@ -256,7 +267,7 @@ private static bool TryParseRetryPolicy(string spec, out RetryPolicy policy) /// internal static bool IsListed(DiscoveredTest test, IReadOnlySet quarantineList) => quarantineList.Contains(test.FullName) - || quarantineList.Contains($"{test.TestClass.FullName}.{test.TestMethod.Name}"); + || quarantineList.Contains($"{test.ClassName}.{test.MethodName}"); private static HashSet LoadQuarantineList(string path) { diff --git a/src/Motus.Cli/Services/ShardSelector.cs b/src/Motus.Cli/Services/ShardSelector.cs index 9b04703..d54d93f 100644 --- a/src/Motus.Cli/Services/ShardSelector.cs +++ b/src/Motus.Cli/Services/ShardSelector.cs @@ -11,7 +11,8 @@ public static class ShardSelector /// /// Returns the subset of belonging to shard /// of . The full set is first sorted by a stable key - /// (assembly path + fully qualified name, ordinal) so the order is identical across machines + /// (assembly path + test id, ordinal; the id rather than the name, since a data source can name + /// its rows differently in each process) so the order is identical across machines /// regardless of reflection enumeration order, then assigned round-robin by /// position % total == index - 1. Round-robin over a stable sort spreads slow classes /// across shards without needing timing data. @@ -76,5 +77,5 @@ public static bool TryParse(string spec, out int index, out int total, out strin } private static string SortKey(DiscoveredTest test) => - $"{test.TestClass.Assembly.Location}::{test.FullName}"; + $"{test.AssemblyPath}::{test.TestId}"; } diff --git a/src/Motus.Cli/Services/TestDiscovery.cs b/src/Motus.Cli/Services/TestDiscovery.cs index bf5dc64..ddca080 100644 --- a/src/Motus.Cli/Services/TestDiscovery.cs +++ b/src/Motus.Cli/Services/TestDiscovery.cs @@ -7,130 +7,107 @@ namespace Motus.Cli.Services; /// A test case found in a test assembly. is the data row of one case of a /// data-driven test, or null for a test without one. /// -public sealed record DiscoveredTest(Type TestClass, MethodInfo TestMethod, string FullName, bool IsIgnored, bool Quarantined = false, object?[]? Arguments = null); - -public sealed class TestDiscovery +/// +/// A test found by lives in a test host process of its own assembly, so +/// , and are +/// null and the test is known by its names. One built from reflection metadata runs in this process. +/// +public sealed record DiscoveredTest(Type? TestClass, MethodInfo? TestMethod, string FullName, bool IsIgnored, bool Quarantined = false, object?[]? Arguments = null) { - private static readonly HashSet ClassAttributes = new(StringComparer.Ordinal) - { - "TestClassAttribute", - "TestFixtureAttribute", - }; + /// The full name of the test class. + public string ClassName { get; init; } = TestClass?.FullName ?? string.Empty; - private static readonly HashSet MethodAttributes = new(StringComparer.Ordinal) - { - "TestMethodAttribute", - "TestAttribute", - "FactAttribute", - }; + /// The test method's name, without a data row. + public string MethodName { get; init; } = TestMethod?.Name ?? string.Empty; - // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply. - private static readonly HashSet ExcludedMethodAttributes = new(StringComparer.Ordinal) - { - "TheoryAttribute", - }; + /// The full path of the test assembly. + public string AssemblyPath { get; init; } = TestClass?.Assembly.Location ?? string.Empty; - private static readonly HashSet IgnoreAttributes = new(StringComparer.Ordinal) - { - "IgnoreAttribute", // MSTest + NUnit - }; + /// The test assembly's name, which reporters use as the suite name. + public string AssemblyName { get; init; } = TestClass?.Assembly.GetName().Name ?? string.Empty; - private static readonly HashSet QuarantineAttributes = new(StringComparer.Ordinal) - { - "QuarantineAttribute", - }; + /// + /// Identifies the test apart from its name: the class, the method with its parameter types, and + /// for a data row, its data source and its place in it. A data source can name its rows differently + /// each time it is read, so sharding keys on this rather than on the name. + /// + public string TestId { get; init; } = FullName; - public List Discover(string[] assemblyPaths, string? filter) + /// The test host that discovered this test and runs it, or null to run it in this process. + internal ITestAssemblyHost? Host { get; init; } +} + +/// +/// Finds the tests in test assemblies. Each assembly is loaded by a test host process of its own, +/// started from the assembly's output folder with its own dependencies, which stays up to run the +/// tests and is shut down when this is disposed. +/// +public sealed class TestDiscovery : IAsyncDisposable +{ + private readonly List _hosts = []; + private readonly List _failedAssemblies = []; + + /// + /// The assemblies passed to that could not be loaded, or whose test + /// host could not start or exited during discovery. motus run fails the run for them. + /// + public IReadOnlyList FailedAssemblies => _failedAssemblies; + + /// + /// The tests in whose full name contains , + /// ignoring case. An assembly that cannot be loaded is reported on standard error, skipped, and + /// listed in . + /// + public async Task> DiscoverAsync(string[] assemblyPaths, string? filter, CancellationToken cancellationToken = default) { var tests = new List(); foreach (var path in assemblyPaths) { var fullPath = Path.GetFullPath(path); - Assembly assembly; + TestHostClient? host = null; try { - assembly = Assembly.LoadFrom(fullPath); - } - catch (Exception ex) - { - Console.Error.WriteLine($"Failed to load assembly {fullPath}: {ex.Message}"); - continue; - } + host = await TestHostClient.StartAsync(fullPath, new TestHostOptions { OnLog = Console.Error.WriteLine }, cancellationToken); + var discovery = await host.DiscoverAsync(filter, cancellationToken); - foreach (var type in assembly.GetExportedTypes()) - { - if (!IsTestClass(type)) + if (discovery.Tests.Count == 0) + { + await host.DisposeAsync(); continue; + } - var classIgnored = type.GetCustomAttributes(true) - .Any(a => IgnoreAttributes.Contains(a.GetType().Name)); - - var classQuarantined = type.GetCustomAttributes(true) - .Any(a => QuarantineAttributes.Contains(a.GetType().Name)); - - foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance)) + _hosts.Add(host); + foreach (var test in discovery.Tests) { - if (!IsTestMethod(method)) - continue; - - var methodIgnored = classIgnored || IsIgnoredMethod(method); - var quarantined = classQuarantined || IsQuarantinedMethod(method); - - foreach (var testCase in TestCaseExpander.Expand(type, method, message => Console.Error.WriteLine(message))) + tests.Add(new DiscoveredTest(null, null, test.FullName, test.IsIgnored, test.Quarantined) { - if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) - continue; - - var isIgnored = methodIgnored || testCase.IgnoreReason is not null; - tests.Add(new DiscoveredTest(type, method, testCase.FullName, isIgnored, quarantined, testCase.Arguments)); - } + ClassName = test.ClassName, + MethodName = test.MethodName, + AssemblyPath = fullPath, + AssemblyName = discovery.AssemblyName, + TestId = test.Id, + Host = host, + }); } } + catch (TestHostException ex) + { + Console.Error.WriteLine($"Failed to load assembly {fullPath}: {ex.Message}"); + _failedAssemblies.Add(fullPath); + if (host is not null) + await host.DisposeAsync(); + } } return tests; } - private static bool IsTestClass(Type type) - { - if (type.IsAbstract || !type.IsClass) - return false; - - return type.GetCustomAttributes(true) - .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes)); - } - - private static bool IsTestMethod(MethodInfo method) - { - return method.GetCustomAttributes(true) - .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes) - && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes)); - } - - private static bool IsIgnoredMethod(MethodInfo method) - { - var attrs = method.GetCustomAttributes(true); - - // MSTest [Ignore] / NUnit [Ignore] - if (attrs.Any(a => IgnoreAttributes.Contains(a.GetType().Name))) - return true; - - // xUnit [Fact(Skip = "reason")] -- check for non-null Skip property on FactAttribute - var factAttr = attrs.FirstOrDefault(a => a.GetType().Name == "FactAttribute"); - if (factAttr is not null) - { - var skipProp = factAttr.GetType().GetProperty("Skip"); - if (skipProp?.GetValue(factAttr) is string) - return true; - } - - return false; - } - - private static bool IsQuarantinedMethod(MethodInfo method) + /// Shuts down the test host processes. + public async ValueTask DisposeAsync() { - return method.GetCustomAttributes(true) - .Any(a => QuarantineAttributes.Contains(a.GetType().Name)); + foreach (var host in _hosts) + await host.DisposeAsync(); + _hosts.Clear(); } } diff --git a/src/Motus.Cli/Services/TestRunner.cs b/src/Motus.Cli/Services/TestRunner.cs index ea71cb2..1e99603 100644 --- a/src/Motus.Cli/Services/TestRunner.cs +++ b/src/Motus.Cli/Services/TestRunner.cs @@ -27,8 +27,8 @@ public enum RetryPolicy public sealed class TestRunner(int maxWorkers) { - private readonly ConcurrentDictionary _initializedAssemblies = new(); - private readonly ConcurrentDictionary _initializedClasses = new(); + // Tests built from reflection metadata rather than discovered run in this process, one host per assembly. + private readonly ConcurrentDictionary _inProcessHosts = new(); public Task RunAsync(List tests, IReporter reporter) => RunAsync(tests, reporter, a11yMode: null, enforcePerfBudget: false, coverageReporters: null); @@ -52,8 +52,8 @@ public async Task RunAsync( { if (maxRetries < 0) maxRetries = 0; - var suiteName = tests.Count > 0 - ? tests[0].TestClass.Assembly.GetName().Name ?? "Motus Tests" + var suiteName = tests.Count > 0 && !string.IsNullOrEmpty(tests[0].AssemblyName) + ? tests[0].AssemblyName : "Motus Tests"; await reporter.OnTestRunStartAsync( @@ -61,23 +61,18 @@ await reporter.OnTestRunStartAsync( var sw = Stopwatch.StartNew(); - // Run [AssemblyInitialize] for each assembly represented in this batch - var failedAssemblies = new HashSet(); - var assemblies = tests.Select(t => t.TestClass.Assembly).Distinct().ToList(); - foreach (var assembly in assemblies) + // Run [AssemblyInitialize] for each assembly represented in this batch, in the test host + // process that assembly runs in + var failedHosts = new HashSet(); + var hosts = tests.Select(HostFor).Distinct().ToList(); + foreach (var host in hosts) { - if (_initializedAssemblies.TryAdd(assembly, true)) + var failure = await host.InitializeAsync(); + if (failure is not null) { - try - { - await RunAssemblyInitializeAsync(assembly); - } - catch (Exception ex) - { - failedAssemblies.Add(assembly); - var msg = ex.InnerException?.Message ?? ex.Message; - Console.Error.WriteLine($"AssemblyInitialize failed for {assembly.GetName().Name}: {msg}"); - } + failedHosts.Add(host); + var assemblyName = tests.First(t => HostFor(t) == host).AssemblyName; + Console.Error.WriteLine($"AssemblyInitialize failed for {assemblyName}: {failure.Message}"); } } @@ -86,6 +81,22 @@ await reporter.OnTestRunStartAsync( var collectedCoverage = new List(); var lockObj = new object(); + // Tests finish on whichever worker gets there, and the reporters keep plain lists, so their + // per-test callbacks are made one at a time. + var reporterGate = new SemaphoreSlim(1, 1); + async Task ReportAsync(Func callback) + { + await reporterGate.WaitAsync(); + try + { + await callback(); + } + finally + { + reporterGate.Release(); + } + } + var tasks = tests.Select(async test => { // Skip ignored tests entirely, without firing reporter events @@ -96,16 +107,14 @@ await reporter.OnTestRunStartAsync( try { var testInfo = new TestInfo(test.FullName, suiteName); - await reporter.OnTestStartAsync(testInfo); + await ReportAsync(() => reporter.OnTestStartAsync(testInfo)); CliTestResult result; var attempts = 1; + TestCaseReport? report = null; - if (failedAssemblies.Contains(test.TestClass.Assembly)) + if (failedHosts.Contains(HostFor(test))) { - AccessibilityViolationSink.Begin(); - PerformanceMetricsSink.Begin(); - CoverageSink.Begin(); result = new CliTestResult(test.FullName, false, TimeSpan.Zero, "AssemblyInitialize failed", null); } @@ -113,37 +122,28 @@ await reporter.OnTestRunStartAsync( { var totalAttempts = maxRetries + 1; var attempt = 0; - Exception? failure; while (true) { attempt++; - AccessibilityViolationSink.Begin(); - PerformanceMetricsSink.Begin(); - CoverageSink.Begin(); - // Each call rebuilds the test instance (and its browser context), - // so every retry runs against clean state. - (result, failure) = await ExecuteTestAsync(test); + // so every retry runs against clean state. What the failed attempts + // collected is discarded: only the last attempt's report is kept. + report = await ExecuteTestAsync(test); + result = new CliTestResult(test.FullName, report.Passed, report.Duration, report.ErrorMessage, report.StackTrace); - if (result.Passed || attempt >= totalAttempts || !ShouldRetry(failure, retryPolicy)) + if (result.Passed || attempt >= totalAttempts || !ShouldRetry(report, retryPolicy)) break; - // Discard sink data captured during the failed attempt; a clean - // Begin() runs at the top of the next iteration. - AccessibilityViolationSink.End(); - PerformanceMetricsSink.End(); - CoverageSink.End(); - WriteRetryNotice(test.FullName, attempt + 1, totalAttempts, retryPolicy); } attempts = attempt; } - var violations = AccessibilityViolationSink.End(); - var perfMetrics = PerformanceMetricsSink.End(); - var coverage = CoverageSink.End(); + var violations = TestCaseReportExtras.Violations(report); + var perfMetrics = TestCaseReportExtras.PerformanceMetrics(report); + var coverage = TestCaseReportExtras.Coverage(report); if (coverage is not null) { @@ -152,30 +152,36 @@ await reporter.OnTestRunStartAsync( collectedCoverage.Add(coverage); } - if (coverageReporters is not null) + await ReportAsync(async () => { - foreach (var cr in coverageReporters) + if (coverageReporters is not null) { - try { await cr.OnCoverageCollectedAsync(coverage, testInfo); } - catch { } + foreach (var cr in coverageReporters) + { + try { await cr.OnCoverageCollectedAsync(coverage, testInfo); } + catch { } + } } - } - if (reporter is ICoverageReporter testReporterCoverage) - { - try { await testReporterCoverage.OnCoverageCollectedAsync(coverage, testInfo); } - catch { } - } + if (reporter is ICoverageReporter testReporterCoverage) + { + try { await testReporterCoverage.OnCoverageCollectedAsync(coverage, testInfo); } + catch { } + } + }); } // Dispatch violations to IAccessibilityReporter reporters if (violations.Count > 0 && reporter is IAccessibilityReporter a11yReporter) { - foreach (var violation in violations) + await ReportAsync(async () => { - try { await a11yReporter.OnAccessibilityViolationAsync(violation, testInfo); } - catch { } - } + foreach (var violation in violations) + { + try { await a11yReporter.OnAccessibilityViolationAsync(violation, testInfo); } + catch { } + } + }); } // Dispatch performance metrics to IPerformanceReporter reporters @@ -186,8 +192,11 @@ await reporter.OnTestRunStartAsync( if (reporter is IPerformanceReporter perfReporter) { - try { await perfReporter.OnPerformanceMetricsCollectedAsync(perfMetrics, budgetResult, testInfo); } - catch { } + await ReportAsync(async () => + { + try { await perfReporter.OnPerformanceMetricsCollectedAsync(perfMetrics, budgetResult, testInfo); } + catch { } + }); } // In enforce mode, override a passing test to failed when budget is exceeded @@ -234,7 +243,7 @@ await reporter.OnTestRunStartAsync( result.FullName, result.Passed, result.Duration.TotalMilliseconds, result.ErrorMessage, result.StackTrace, Flaky: result.Flaky, Quarantined: result.Quarantined, Attempts: result.Attempts); - await reporter.OnTestEndAsync(testInfo, absResult); + await ReportAsync(() => reporter.OnTestEndAsync(testInfo, absResult)); } finally { @@ -296,32 +305,9 @@ await reporter.OnTestRunStartAsync( } } - // Run [ClassCleanup] for all initialized classes - foreach (var type in _initializedClasses.Keys) - { - try - { - await RunStaticLifecycleMethodAsync(type, "ClassCleanupAttribute"); - await RunStaticLifecycleMethodAsync(type, "OneTimeTearDownAttribute"); - } - catch (Exception ex) - { - Console.Error.WriteLine($"ClassCleanup failed for {type.FullName}: {ex.InnerException?.Message ?? ex.Message}"); - } - } - - // Run [AssemblyCleanup] for each initialized assembly - foreach (var assembly in assemblies) - { - try - { - await RunAssemblyCleanupAsync(assembly); - } - catch (Exception ex) - { - Console.Error.WriteLine($"AssemblyCleanup failed for {assembly.GetName().Name}: {ex.InnerException?.Message ?? ex.Message}"); - } - } + // Run [ClassCleanup] for each class that ran a test, then [AssemblyCleanup], assembly by assembly + foreach (var host in hosts) + await host.CleanupAsync(); var skipped = tests.Count(t => t.IsIgnored); // Quarantined results occupy their own bucket and are excluded from passed/failed @@ -370,94 +356,26 @@ private static void WriteRetryNotice(string testName, int attempt, int totalAtte /// retries any failure; /// retries only transient CDP disconnects. /// - private static bool ShouldRetry(Exception? failure, RetryPolicy policy) + private static bool ShouldRetry(TestCaseReport report, RetryPolicy policy) { - if (failure is null) + if (report.Passed) return false; - return policy == RetryPolicy.Flake || IsTransientCdpFailure(failure); + return policy == RetryPolicy.Flake || report.TransientFailure; } /// - /// Returns true when the exception chain contains a transient CDP failure that - /// is safe to recover from by re-running the entire test (which rebuilds a fresh - /// browser context and WebSocket). Used to drive --retries. + /// The host that runs : the test host process that discovered it, or for a + /// test built from reflection metadata, one in this process for its assembly. /// - private static bool IsTransientCdpFailure(Exception? ex) - { - if (Abstractions.BrowserFailure.IsBrowserLost(ex)) - return true; - - // The transport's own disconnect reaches the runner unwrapped when it escapes from outside - // a command, where nothing has turned it into the public failure yet. - for (var current = ex; current is not null; current = current.InnerException) - { - if (current is CdpDisconnectedException) - return true; - } - - return false; - } - - private async Task<(CliTestResult Result, Exception? Failure)> ExecuteTestAsync(DiscoveredTest test) - { - var run = await TestCaseInvoker.RunAsync( - test.TestClass, - test.TestMethod, - test.Arguments, - DisplayNameOf(test), - beforeInstance: async () => - { - // Run [ClassInitialize] once per type (static, takes TestContext) - if (_initializedClasses.TryAdd(test.TestClass, true)) - { - await RunStaticLifecycleMethodAsync(test.TestClass, "ClassInitializeAttribute"); - await RunStaticLifecycleMethodAsync(test.TestClass, "OneTimeSetUpAttribute"); - } - }, - onCleanupFailure: ex => Console.Error.WriteLine($"TestCleanup failed for {test.FullName}: {ex.Message}")); - - return run.Failure is null - ? (new CliTestResult(test.FullName, true, run.Duration, null, null), null) - : (new CliTestResult(test.FullName, false, run.Duration, run.Failure.Message, run.Failure.StackTrace), run.Failure); - } - - /// The test case's name without its class, such as SignsIn ("owner"). - private static string DisplayNameOf(DiscoveredTest test) - { - var prefix = test.TestClass.FullName + "."; - return test.FullName.StartsWith(prefix, StringComparison.Ordinal) ? test.FullName[prefix.Length..] : test.TestMethod.Name; - } - - private static async Task RunAssemblyCleanupAsync(Assembly assembly) - { - foreach (var type in assembly.GetExportedTypes()) - { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyCleanupAttribute")); - - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); - } - } - - private static async Task RunAssemblyInitializeAsync(Assembly assembly) - { - foreach (var type in assembly.GetExportedTypes()) - { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyInitializeAttribute")); - - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); - } - } + private ITestAssemblyHost HostFor(DiscoveredTest test) + => test.Host ?? _inProcessHosts.GetOrAdd(test.TestClass!.Assembly, assembly => new InProcessTestHost(assembly, log: Console.Error.WriteLine)); - private static async Task RunStaticLifecycleMethodAsync(Type type, string attributeName) + private Task ExecuteTestAsync(DiscoveredTest test) { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); + if (test.Host is { } host) + return host.RunAsync(test.TestId, test.FullName); - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); + var testCase = new DiscoveredTestCase(test.TestClass!, test.TestMethod!, test.FullName, test.Arguments, test.IsIgnored, null, test.Quarantined, test.TestId); + return ((InProcessTestHost)HostFor(test)).RunAsync(testCase); } } diff --git a/src/Motus.Runner/Components/Explorer/TestMethodNode.razor b/src/Motus.Runner/Components/Explorer/TestMethodNode.razor index aa1c619..ae17241 100644 --- a/src/Motus.Runner/Components/Explorer/TestMethodNode.razor +++ b/src/Motus.Runner/Components/Explorer/TestMethodNode.razor @@ -32,9 +32,9 @@ // A data row's name carries its arguments, e.g. SignsIn ("owner"), so rows of one method differ. private string MethodDisplayName => - Test.TestClass?.FullName is { } className && Test.FullName.StartsWith(className + ".", StringComparison.Ordinal) + (Test.TestClass?.FullName ?? Test.ClassFullName) is { } className && Test.FullName.StartsWith(className + ".", StringComparison.Ordinal) ? Test.FullName[(className.Length + 1)..] - : Test.TestMethod?.Name ?? GetMethodFromFullName(Test.FullName); + : Test.TestMethod?.Name ?? Test.MethodName ?? GetMethodFromFullName(Test.FullName); private bool ShowRerunButton => Options.ViewerMode == ViewerMode.Trx && CurrentStatus != TestStatus.Passed; diff --git a/src/Motus.Runner/Program.cs b/src/Motus.Runner/Program.cs index 11c4c51..957e6ad 100644 --- a/src/Motus.Runner/Program.cs +++ b/src/Motus.Runner/Program.cs @@ -1,3 +1,15 @@ using Motus.Runner; +using Motus.Runner.Services.Execution; + +// motus run starts this assembly as the test host for each test assembly (see TestHostServer). +if (TestHostServer.IsHostCommand(args)) +{ + TestHostServer.Prepare(args); + var exitCode = await TestHostServer.RunAsync(args); + + // A test can leave a foreground thread running, such as a server it never stopped, which + // would otherwise keep the process alive after the runner has gone. + Environment.Exit(exitCode); +} await RunnerHost.StartAsync(args); diff --git a/src/Motus.Runner/Services/Execution/ITestAssemblyHost.cs b/src/Motus.Runner/Services/Execution/ITestAssemblyHost.cs new file mode 100644 index 0000000..cd0399f --- /dev/null +++ b/src/Motus.Runner/Services/Execution/ITestAssemblyHost.cs @@ -0,0 +1,108 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Motus.Abstractions; + +namespace Motus.Runner.Services.Execution; + +/// +/// Runs the tests of one test assembly: its [AssemblyInitialize], each test case on request, +/// then its class and assembly cleanup. runs them in this process; +/// runs them in a test host process of the assembly's own. +/// +internal interface ITestAssemblyHost : IAsyncDisposable +{ + /// + /// Runs the assembly's [AssemblyInitialize] methods, once for the life of the host. + /// Returns what went wrong, or null when they ran or there are none. + /// + Task InitializeAsync(CancellationToken cancellationToken = default); + + /// + /// Runs the discovered test case once, or when no case has that id, the + /// one named . Cancelling cancels + /// the token the test reads from its TestContext; the test still runs to its end. + /// + Task RunAsync(string testId, string fullName, CancellationToken cancellationToken = default); + + /// + /// Runs [ClassCleanup] and NUnit [OneTimeTearDown] for each class that ran a test, + /// then the assembly's [AssemblyCleanup] methods. + /// + Task CleanupAsync(CancellationToken cancellationToken = default); +} + +/// What discovery reports about one test case. +/// The test class's full name. +/// The test method's name, without the data row. +/// The case's identity apart from its name, which runs ask for it by. +internal sealed record TestCaseInfo( + string FullName, + string ClassName, + string MethodName, + bool IsIgnored, + string? IgnoreReason, + bool Quarantined, + string Id); + +/// An exception from test code, as text, so that it can cross a process boundary. +internal sealed record TestHostFailure(string Message, string? StackTrace); + +/// +/// The outcome of one run of one test case, with what the run collected for the reporters. +/// +/// +/// What the run collected is carried as JSON rather than as Motus types: the test host runs on the +/// suite's Motus, whose types can differ from the runner's. reads +/// it back into the runner's own. +/// +/// True when the failure was a lost browser connection, which the +/// default retry policy re-runs. +internal sealed record TestCaseReport( + string FullName, + bool Passed, + double DurationMs, + string? ErrorMessage, + string? StackTrace, + bool TransientFailure = false, + JsonElement? Violations = null, + JsonElement? PerformanceMetrics = null, + JsonElement? Coverage = null) +{ + [JsonIgnore] + public TimeSpan Duration => TimeSpan.FromMilliseconds(DurationMs); + + /// A failed run that never reached the test. + public static TestCaseReport NotRun(string fullName, string message) + => new(fullName, false, 0, message, null); +} + +/// +/// Reads what a test run collected out of its , in the runner. Anything +/// that cannot be read is left out rather than failing the run. +/// +internal static class TestCaseReportExtras +{ + public static IReadOnlyList Violations(TestCaseReport? report) + => Read>(report?.Violations) ?? []; + + public static PerformanceMetrics? PerformanceMetrics(TestCaseReport? report) + => Read(report?.PerformanceMetrics); + + public static CoverageData? Coverage(TestCaseReport? report) + => Read(report?.Coverage); + + private static T? Read(JsonElement? json) where T : class + { + if (json is not { ValueKind: not JsonValueKind.Null and not JsonValueKind.Undefined } element) + return null; + + try + { + return element.Deserialize(TestHostJson.Options); + } + catch (JsonException) + { + return null; + } + } +} diff --git a/src/Motus.Runner/Services/Execution/InProcessTestHost.cs b/src/Motus.Runner/Services/Execution/InProcessTestHost.cs new file mode 100644 index 0000000..5686f59 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/InProcessTestHost.cs @@ -0,0 +1,181 @@ +using System.Collections.Concurrent; +using System.Reflection; + +namespace Motus.Runner.Services.Execution; + +/// +/// Runs the tests of an assembly loaded into this process, by reflection. The test host process +/// runs one of these for the assembly it was started for, and the runners' own tests use one +/// directly. +/// +internal sealed class InProcessTestHost : ITestAssemblyHost +{ + private readonly Assembly _assembly; + private readonly ConcurrentDictionary _cases = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _initializedClasses = new(); + private readonly Action _log; + private int _initialized; + + /// Told about cleanup methods that throw, which do not fail a test. + public InProcessTestHost(Assembly assembly, IEnumerable? cases = null, Action? log = null) + { + _assembly = assembly; + _log = log ?? (_ => { }); + if (cases is not null) + { + foreach (var testCase in cases) + _cases[testCase.Id] = testCase; + } + } + + public async Task InitializeAsync(CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref _initialized, 1) != 0) + return null; + + try + { + // [AssemblyInitialize] is a static method that takes TestContext + // Look across all types in the assembly for it + foreach (var type in _assembly.GetExportedTypes()) + { + foreach (var method in StaticMethodsWith(type, "AssemblyInitializeAttribute", BindingFlags.Public | BindingFlags.Static)) + await TestCaseInvoker.InvokeStaticAsync(method, type); + } + + return null; + } + catch (Exception ex) + { + return new TestHostFailure(ex.InnerException?.Message ?? ex.Message, ex.InnerException?.StackTrace ?? ex.StackTrace); + } + } + + public Task RunAsync(string testId, string fullName, CancellationToken cancellationToken = default) + { + // By name first: after a rebuild that removed or reordered a [DataRow], an old row id can + // point at a different row, while a name that matches one case is that case. The id settles + // what the name can't: rows whose names change on every read of their data source. + var named = _cases.Values.Where(c => c.FullName == fullName).Take(2).ToList(); + if (named.Count == 1) + return RunAsync(named[0], cancellationToken); + + // A case whose name never changes but no longer matches is not the case that was listed: + // a [DataRow] that moved up after one before it was removed, say. + if (_cases.TryGetValue(testId, out var testCase)) + { + if (!testCase.ConstantName || testCase.FullName == fullName) + return RunAsync(testCase, cancellationToken); + + return Task.FromResult(TestCaseReport.NotRun(fullName, + $"Test {fullName} is no longer in {_assembly.GetName().Name}: its data rows have changed since the tests were listed. " + + "Load the tests again to run the current ones.")); + } + + return Task.FromResult(TestCaseReport.NotRun(fullName, DescribeMissing(testId, fullName))); + } + + private string DescribeMissing(string testId, string fullName) + { + if (TestCaseExpander.TryParseRowId(testId, out var methodId, out var sourceIndex, out var rowIndex)) + { + var rows = _cases.Values.Count(c => TestCaseExpander.TryParseRowId(c.Id, out var m, out var s, out _) + && m == methodId && s == sourceIndex); + if (rows > 0 || _cases.Values.Any(c => c.Id.StartsWith(methodId, StringComparison.Ordinal))) + { + return $"Test {fullName} was not run: it was row {rowIndex + 1} of its data source, " + + $"which now returns {rows} row(s)."; + } + } + + return $"Test {fullName} was not found in {_assembly.GetName().Name}. It may have been renamed or removed."; + } + + /// Runs once, collecting what the sinks gathered while it ran. + public async Task RunAsync(DiscoveredTestCase testCase, CancellationToken cancellationToken = default) + { + MotusInterop.BeginCollecting(); + + var run = await TestCaseInvoker.RunAsync( + testCase.TestClass, + testCase.TestMethod, + testCase.Arguments, + DisplayNameOf(testCase), + beforeInstance: async () => + { + // Run [ClassInitialize] / [OneTimeSetUp] once per class, before its first test + if (_initializedClasses.TryAdd(testCase.TestClass, true)) + { + await RunStaticLifecycleMethodsAsync(testCase.TestClass, "ClassInitializeAttribute"); + await RunStaticLifecycleMethodsAsync(testCase.TestClass, "OneTimeSetUpAttribute"); + } + }, + onCleanupFailure: ex => _log($"TestCleanup failed for {testCase.FullName}: {ex.Message}"), + cancellationToken: cancellationToken); + + var (violations, metrics, coverage) = MotusInterop.EndCollecting(); + + return new TestCaseReport( + testCase.FullName, + run.Failure is null, + run.Duration.TotalMilliseconds, + run.Failure?.Message, + run.Failure?.StackTrace, + MotusInterop.IsBrowserLost(run.Failure), + violations, + metrics, + coverage); + } + + public async Task CleanupAsync(CancellationToken cancellationToken = default) + { + // Run [ClassCleanup] / [OneTimeTearDown] for each class that ran a test + foreach (var type in _initializedClasses.Keys.ToList()) + { + if (!_initializedClasses.TryRemove(type, out _)) + continue; + + try + { + await RunStaticLifecycleMethodsAsync(type, "ClassCleanupAttribute"); + await RunStaticLifecycleMethodsAsync(type, "OneTimeTearDownAttribute"); + } + catch (Exception ex) + { + _log($"ClassCleanup failed for {type.FullName}: {ex.InnerException?.Message ?? ex.Message}"); + } + } + + // Run [AssemblyCleanup], which is where suites close their browsers + try + { + foreach (var type in _assembly.GetExportedTypes()) + { + foreach (var method in StaticMethodsWith(type, "AssemblyCleanupAttribute", BindingFlags.Public | BindingFlags.Static)) + await TestCaseInvoker.InvokeStaticAsync(method, type); + } + } + catch (Exception ex) + { + _log($"AssemblyCleanup failed for {_assembly.GetName().Name}: {ex.InnerException?.Message ?? ex.Message}"); + } + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + /// The test case's name without its class, such as SignsIn ("owner"). + private static string DisplayNameOf(DiscoveredTestCase testCase) + { + var prefix = testCase.TestClass.FullName + "."; + return testCase.FullName.StartsWith(prefix, StringComparison.Ordinal) ? testCase.FullName[prefix.Length..] : testCase.TestMethod.Name; + } + + private static async Task RunStaticLifecycleMethodsAsync(Type type, string attributeName) + { + foreach (var method in StaticMethodsWith(type, attributeName, BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)) + await TestCaseInvoker.InvokeStaticAsync(method, type); + } + + private static IEnumerable StaticMethodsWith(Type type, string attributeName, BindingFlags flags) + => type.GetMethods(flags).Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); +} diff --git a/src/Motus.Runner/Services/Execution/MotusInterop.cs b/src/Motus.Runner/Services/Execution/MotusInterop.cs new file mode 100644 index 0000000..f9c3aa4 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/MotusInterop.cs @@ -0,0 +1,172 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Motus.Abstractions; + +namespace Motus.Runner.Services.Execution; + +/// +/// Every call the test host makes into Motus, each one guarded. +/// +/// +/// The test host runs a suite on the Motus its project references, which can be older (or newer) +/// than the one this runner was built against. A member the runner uses that the suite's Motus +/// does not have fails when the method that names it is compiled, so each such call is made from +/// a method of its own that is never inlined, from inside a guard. A call that cannot be made is +/// reported once and then skipped: the test still runs and reports its own result, and only what +/// the call would have added (an accessibility report, a performance metric, coverage) is missing. +/// +internal static class MotusInterop +{ + private static readonly ConcurrentDictionary s_unavailable = new(StringComparer.Ordinal); + + /// Told once about each feature the suite's Motus cannot provide. + public static Action? Warn { get; set; } + + /// The version of the Motus assembly this process runs on, or null when there is none. + public static Version? LoadedMotusVersion() + { + try + { + return Assembly.Load(new AssemblyName("Motus")).GetName().Version; + } + catch + { + return null; + } + } + + /// Publishes the running test method's name to Motus (for [PerformanceBudget] and artifact names). + public static void SetTestMethodName(string name) => Guard("test method names", () => SetTestMethodNameCore(name)); + + /// Clears what published. + public static void ClearTestMethodName() => Guard("test method names", ClearTestMethodNameCore); + + /// Starts collecting accessibility violations, performance metrics and coverage for this async flow. + public static void BeginCollecting() + { + Guard("accessibility reports", BeginViolationsCore); + Guard("performance metrics", BeginMetricsCore); + Guard("coverage", BeginCoverageCore); + } + + /// + /// Ends what started and returns what was collected as JSON, so that + /// it crosses to the runner without naming the suite's Motus types. + /// + public static (JsonElement? Violations, JsonElement? Metrics, JsonElement? Coverage) EndCollecting() + { + JsonElement? violations = null, metrics = null, coverage = null; + Guard("accessibility reports", () => violations = EndViolationsCore()); + Guard("performance metrics", () => metrics = EndMetricsCore()); + Guard("coverage", () => coverage = EndCoverageCore()); + return (violations, metrics, coverage); + } + + /// + /// True when the exception chain says the browser connection was lost, which the default retry + /// policy re-runs. Falls back to the exceptions' type names on a Motus without these types. + /// + public static bool IsBrowserLost(Exception? exception) + { + if (exception is null) + return false; + + var lost = false; + if (Guard("lost browser detection", () => lost = IsBrowserLostCore(exception))) + return lost; + + for (var current = exception; current is not null; current = current.InnerException) + { + if (current.GetType().Name is "MotusTargetClosedException" or "CdpDisconnectedException") + return true; + } + + return false; + } + + /// Reads motus.config.json now, so that it is read from the current directory. + public static void LoadConfig() => Guard("motus.config.json", LoadConfigCore); + + /// + /// Runs unless is known to be missing. Returns + /// false, and says so once, when the suite's Motus does not have what it needs. + /// + internal static bool Guard(string feature, Action call) + { + if (s_unavailable.ContainsKey(feature)) + return false; + + try + { + call(); + return true; + } + catch (Exception ex) when (IsMissingFromMotus(ex)) + { + if (s_unavailable.TryAdd(feature, true)) + { + var version = LoadedMotusVersion()?.ToString(3) ?? "unknown"; + Warn?.Invoke($"The Motus {version} this suite references has no support for {feature} that this motus can use, " + + $"so the run goes on without it. ({ex.GetType().Name}: {ex.Message})"); + } + + return false; + } + } + + /// The exceptions the runtime throws when compiled code names something an assembly lacks. + internal static bool IsMissingFromMotus(Exception ex) + => ex is TypeLoadException or MissingMemberException or FileNotFoundException or FileLoadException or BadImageFormatException; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void SetTestMethodNameCore(string name) => TestMethodNameContext.Set(name); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ClearTestMethodNameCore() => TestMethodNameContext.Clear(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BeginViolationsCore() => AccessibilityViolationSink.Begin(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BeginMetricsCore() => PerformanceMetricsSink.Begin(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BeginCoverageCore() => CoverageSink.Begin(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static JsonElement? EndViolationsCore() + { + var violations = AccessibilityViolationSink.End(); + return violations.Count > 0 ? JsonSerializer.SerializeToElement(violations, TestHostJson.Options) : null; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static JsonElement? EndMetricsCore() + => PerformanceMetricsSink.End() is { } metrics ? JsonSerializer.SerializeToElement(metrics, TestHostJson.Options) : null; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static JsonElement? EndCoverageCore() + => CoverageSink.End() is { } coverage ? JsonSerializer.SerializeToElement(coverage, TestHostJson.Options) : null; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool IsBrowserLostCore(Exception exception) + { + if (BrowserFailure.IsBrowserLost(exception)) + return true; + + // The transport's own disconnect reaches the runner unwrapped when it escapes from outside + // a command, where nothing has turned it into the public failure yet. + for (var current = exception; current is not null; current = current.InnerException) + { + if (current is CdpDisconnectedException) + return true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void LoadConfigCore() => _ = MotusConfigLoader.Config; +} diff --git a/src/Motus.Runner/Services/Execution/TestCaseDiscovery.cs b/src/Motus.Runner/Services/Execution/TestCaseDiscovery.cs new file mode 100644 index 0000000..b1d1dea --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestCaseDiscovery.cs @@ -0,0 +1,155 @@ +using System.Reflection; + +namespace Motus.Runner.Services.Execution; + +/// +/// A test case found in a test assembly loaded into this process, with what it takes to run it. +/// +/// The name the runners report and filter on. +/// The data row, or null for a test method without one. +/// Why the test is ignored, when it is and a reason was given. +/// True when the test or its class carries [Quarantine]. +/// The case's identity apart from its name (see ). +/// Defaults to for a case built by hand. +internal sealed record DiscoveredTestCase( + Type TestClass, + MethodInfo TestMethod, + string FullName, + object?[]? Arguments, + bool IsIgnored, + string? IgnoreReason, + bool Quarantined, + string? Id = null, + bool ConstantName = true) +{ + /// The case's identity, which the runner asks for it by. + public string Id { get; init; } = Id ?? FullName; + + /// What a runner in another process is told about this test case. + public TestCaseInfo ToInfo() + => new(FullName, TestClass.FullName ?? TestClass.Name, TestMethod.Name, IsIgnored, IgnoreReason, Quarantined, Id); +} + +/// +/// Finds the tests in a loaded test assembly: MSTest, NUnit and xUnit test classes and methods, +/// expanded into one test case per data row. Runs in the test host process, for both runners. +/// +internal static class TestCaseDiscovery +{ + private static readonly HashSet ClassAttributes = new(StringComparer.Ordinal) + { + "TestClassAttribute", + "TestFixtureAttribute", + }; + + private static readonly HashSet MethodAttributes = new(StringComparer.Ordinal) + { + "TestMethodAttribute", + "TestAttribute", + "FactAttribute", + }; + + // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply. + private static readonly HashSet ExcludedMethodAttributes = new(StringComparer.Ordinal) + { + "TheoryAttribute", + }; + + private static readonly HashSet IgnoreAttributes = new(StringComparer.Ordinal) + { + "IgnoreAttribute", // MSTest + NUnit + }; + + private static readonly HashSet QuarantineAttributes = new(StringComparer.Ordinal) + { + "QuarantineAttribute", + }; + + /// + /// The test cases in whose full name contains + /// (ignoring case), or all of them when it is null. is told about a + /// data source that could not be read. + /// + public static List Discover(Assembly assembly, string? filter, Action? onWarning = null) + { + var tests = new List(); + + foreach (var type in assembly.GetExportedTypes()) + { + if (!IsTestClass(type)) + continue; + + var classAttributes = type.GetCustomAttributes(true); + var (classIgnored, classReason) = GetIgnoreInfo(classAttributes); + var classQuarantined = classAttributes.Any(a => QuarantineAttributes.Contains(a.GetType().Name)); + + var testMethods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance).Where(IsTestMethod).ToList(); + var overloaded = testMethods.GroupBy(m => m.Name).Where(g => g.Count() > 1).Select(g => g.Key).ToHashSet(StringComparer.Ordinal); + + foreach (var method in testMethods) + { + + var methodAttributes = method.GetCustomAttributes(true); + var (methodIgnored, methodReason) = GetIgnoreInfo(methodAttributes); + var quarantined = classQuarantined || methodAttributes.Any(a => QuarantineAttributes.Contains(a.GetType().Name)); + + foreach (var expanded in TestCaseExpander.Expand(type, method, onWarning)) + { + // Overloads without data rows would share a name, which reporters and results key on. + var testCase = overloaded.Contains(method.Name) && expanded.Arguments is null + ? expanded with { FullName = $"{type.FullName}.{method.Name}({string.Join(", ", method.GetParameters().Select(p => p.ParameterType.Name))})" } + : expanded; + + if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) + continue; + + var isIgnored = classIgnored || methodIgnored || testCase.IgnoreReason is not null; + var reason = classIgnored ? classReason : methodIgnored ? methodReason : testCase.IgnoreReason; + tests.Add(new DiscoveredTestCase(type, method, testCase.FullName, testCase.Arguments, isIgnored, reason, quarantined, testCase.Id, testCase.ConstantName)); + } + } + } + + return tests; + } + + private static bool IsTestClass(Type type) + { + if (type.IsAbstract || !type.IsClass) + return false; + + return type.GetCustomAttributes(true) + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes)); + } + + private static bool IsTestMethod(MethodInfo method) + { + return method.GetCustomAttributes(true) + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes) + && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes)); + } + + private static (bool IsIgnored, string? Reason) GetIgnoreInfo(object[] attrs) + { + // MSTest [Ignore] / [Ignore("reason")] / NUnit [Ignore("reason")] + var ignoreAttr = attrs.FirstOrDefault(a => IgnoreAttributes.Contains(a.GetType().Name)); + if (ignoreAttr is not null) + { + // MSTest uses IgnoreMessage; NUnit uses Reason + var msg = ignoreAttr.GetType().GetProperty("IgnoreMessage")?.GetValue(ignoreAttr) as string + ?? ignoreAttr.GetType().GetProperty("Reason")?.GetValue(ignoreAttr) as string; + return (true, msg); + } + + // xUnit [Fact(Skip = "reason")] + var factAttr = attrs.FirstOrDefault(a => a.GetType().Name == "FactAttribute"); + if (factAttr is not null) + { + var skip = factAttr.GetType().GetProperty("Skip")?.GetValue(factAttr) as string; + if (skip is not null) + return (true, skip); + } + + return (false, null); + } +} diff --git a/src/Motus.Runner/Services/Execution/TestCaseExpander.cs b/src/Motus.Runner/Services/Execution/TestCaseExpander.cs index b276320..5c9697b 100644 --- a/src/Motus.Runner/Services/Execution/TestCaseExpander.cs +++ b/src/Motus.Runner/Services/Execution/TestCaseExpander.cs @@ -10,7 +10,13 @@ namespace Motus.Runner.Services.Execution; /// The name the runners report and filter on. /// The data row, or null for a test method without one. /// Set when the data row itself is marked ignored. -internal sealed record ExpandedTestCase(string FullName, object?[]? Arguments, string? IgnoreReason); +/// Identifies the test case without depending on its display name: the class, the +/// method with its parameter types, and for a data row, which data source on the method and which +/// row of it. A data source can name its rows differently each time it is read (a Guid or a time in +/// the row), so this is what a runner that discovers again in a new process looks the case up by. +/// True when the name is the same on every read: a test without data +/// rows, or a [DataRow], whose values are constants. Such a case is found by its name only. +internal sealed record ExpandedTestCase(string FullName, object?[]? Arguments, string? IgnoreReason, string Id, bool ConstantName = true); /// /// Turns a test method into its test cases, one per [DataRow], [DynamicData] row or @@ -20,6 +26,8 @@ internal static class TestCaseExpander { private const string TestDataSourceInterfaceName = "Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSource"; + private static readonly HashSet DataRowAttributeNames = new(StringComparer.Ordinal) { "DataRowAttribute" }; + /// /// True when 's type, or a type it derives from, is named in /// , so that [DataTestMethod] and other subclasses of @@ -46,15 +54,20 @@ public static bool IsOrDerivesFrom(object attribute, IReadOnlySet names) public static List Expand(Type testClass, MethodInfo method, Action? onError = null) { var baseName = $"{testClass.FullName}.{method.Name}"; + var methodId = MethodId(testClass, method); var cases = new List(); + var sourceIndex = -1; var seen = new HashSet(StringComparer.Ordinal); foreach (var attribute in method.GetCustomAttributes(true)) { var dataSource = attribute.GetType().GetInterfaces().FirstOrDefault(i => i.FullName == TestDataSourceInterfaceName); + var constantRows = IsOrDerivesFrom(attribute, DataRowAttributeNames); if (dataSource is null) continue; + sourceIndex++; + List rows; try { @@ -72,8 +85,9 @@ public static List Expand(Type testClass, MethodInfo method, A var ignoreReason = attribute.GetType().GetProperty("IgnoreMessage")?.GetValue(attribute) as string; var getDisplayName = dataSource.GetMethod("GetDisplayName"); - foreach (var row in rows) + for (var rowIndex = 0; rowIndex < rows.Count; rowIndex++) { + var row = rows[rowIndex]; var displayName = TryGetDisplayName(getDisplayName, attribute, method, row) ?? DefaultDisplayName(method, row); @@ -87,18 +101,44 @@ public static List Expand(Type testClass, MethodInfo method, A for (var n = 2; !seen.Add(unique); n++) unique = $"{fullName} #{n}"; - cases.Add(new ExpandedTestCase(unique, row, ignoreReason)); + cases.Add(new ExpandedTestCase(unique, row, ignoreReason, RowId(methodId, sourceIndex, rowIndex), constantRows)); } } // No data source, or none that could be read: a single case. One whose method takes // parameters fails when run, saying they were not supplied. if (cases.Count == 0) - cases.Add(new ExpandedTestCase(baseName, null, null)); + cases.Add(new ExpandedTestCase(baseName, null, null, methodId)); return cases; } + /// + /// {Class}.{Method}({parameter types}), which tells overloads apart where the full name does not. + /// + internal static string MethodId(Type testClass, MethodInfo method) + => $"{testClass.FullName}.{method.Name}({string.Join(",", method.GetParameters().Select(p => p.ParameterType.FullName ?? p.ParameterType.Name))})"; + + /// The id of row of the method's data source . + internal static string RowId(string methodId, int sourceIndex, int rowIndex) => $"{methodId}#{sourceIndex}:{rowIndex}"; + + /// Splits a row id into its method id, data source and row, or returns false for a test without rows. + internal static bool TryParseRowId(string id, out string methodId, out int sourceIndex, out int rowIndex) + { + methodId = id; + sourceIndex = rowIndex = -1; + var hash = id.LastIndexOf('#'); + if (hash < 0 || id.IndexOf(')', hash) >= 0) + return false; + + var parts = id[(hash + 1)..].Split(':'); + if (parts.Length != 2 || !int.TryParse(parts[0], out sourceIndex) || !int.TryParse(parts[1], out rowIndex)) + return false; + + methodId = id[..hash]; + return true; + } + private static string? TryGetDisplayName(MethodInfo? getDisplayName, object attribute, MethodInfo method, object?[] row) { if (getDisplayName is null) diff --git a/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs b/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs index a6749df..e0d0f61 100644 --- a/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs +++ b/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using System.Reflection; -using Motus; namespace Motus.Runner.Services.Execution; @@ -51,7 +50,7 @@ public static async Task RunAsync( // Publish the test method name so MotusTestBase can resolve per-method attributes // (e.g. [PerformanceBudget]) and name failure artifacts without a TestContext - TestMethodNameContext.Set(testMethod.Name); + MotusInterop.SetTestMethodName(testMethod.Name); try { if (beforeInstance is not null) @@ -112,7 +111,7 @@ public static async Task RunAsync( if (context is not null) MSTestContextFactory.SetCurrent(contextType!, null); - TestMethodNameContext.Clear(); + MotusInterop.ClearTestMethodName(); } return new TestCaseRunResult(failure, sw.Elapsed); diff --git a/src/Motus.Runner/Services/Execution/TestHostClient.cs b/src/Motus.Runner/Services/Execution/TestHostClient.cs new file mode 100644 index 0000000..30c70d3 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestHostClient.cs @@ -0,0 +1,486 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.IO.Pipes; +using System.Reflection; +using System.Runtime.Versioning; +using System.Text.Json.Nodes; + +namespace Motus.Runner.Services.Execution; + +/// What a runner wants from a test host besides results. +internal sealed class TestHostOptions +{ + /// Stream the browser and record a timeline, for the visual runner. + public bool Visual { get; init; } + + /// Told each warning the test host reports, such as a cleanup method that threw. + public Action? OnLog { get; init; } + + /// Told each screencast frame, as base64 JPEG. + public Action? OnFrame { get; init; } + + /// Told each timeline entry the recorder hook makes. + public Action? OnTimelineEntry { get; init; } + + /// + /// Asked before each browser action when is set; the action waits for the + /// returned task, which is how step debugging holds it. + /// + public Func? OnStep { get; init; } + + /// How long the test host may take to start and connect. + public TimeSpan StartTimeout { get; init; } = TimeSpan.FromSeconds(60); +} + +/// The tests a test host found in its assembly. +internal sealed record TestHostDiscovery(string AssemblyName, IReadOnlyList Tests); + +/// A test host that could not start, could not load its assembly, or exited. +internal sealed class TestHostException(string message) : Exception(message); + +/// +/// The runner's side of a test host process (): starts it for one test +/// assembly, then discovers and runs that assembly's tests in it. Several runs may be in flight +/// at once. When the process exits unexpectedly, the runs in flight and every later one fail with +/// a message that says so, and its own output, including any crash report, has already gone to +/// this process's console, which it shares. +/// +internal sealed class TestHostClient : ITestAssemblyHost +{ + private readonly Process _process; + private readonly TestHostConnection _connection; + private readonly TestHostOptions _options; + private readonly ConcurrentDictionary> _pending = new(); + private readonly Task _readLoop; + private readonly object _initializeLock = new(); + private Task? _initialize; + private volatile string? _exitMessage; + private volatile bool _disposing; + private int _nextId; + + private TestHostClient(string assemblyPath, Process process, NamedPipeServerStream pipe, TestHostOptions options) + { + AssemblyPath = assemblyPath; + _process = process; + _options = options; + _connection = new TestHostConnection(pipe); + _readLoop = Task.Run(ReadLoopAsync); + } + + /// The full path of the test assembly this host runs. + public string AssemblyPath { get; } + + private string FileName => Path.GetFileName(AssemblyPath); + + /// Starts a test host for and waits for it to connect. + /// The process could not be started or exited first. + public static async Task StartAsync(string assemblyPath, TestHostOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new TestHostOptions(); + assemblyPath = Path.GetFullPath(assemblyPath); + if (!File.Exists(assemblyPath)) + throw new TestHostException("The file does not exist."); + + var pipeName = TestHostLauncher.NewPipeName(); + var pipe = new NamedPipeServerStream(pipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous); + + Process process; + try + { + process = Process.Start(TestHostLauncher.CreateStartInfo(pipeName, assemblyPath, options.Visual)) + ?? throw new TestHostException("Could not start a test host process."); + } + catch (Exception ex) when (ex is not TestHostException) + { + await pipe.DisposeAsync(); + throw new TestHostException($"Could not start a test host process: {ex.Message}"); + } + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(options.StartTimeout); + var connected = pipe.WaitForConnectionAsync(timeout.Token); + var exited = process.WaitForExitAsync(timeout.Token); + try + { + await Task.WhenAny(connected, exited); + } + catch + { + // Inspected below. + } + + if (connected.IsCompletedSuccessfully) + return new TestHostClient(assemblyPath, process, pipe, options); + + var message = process.HasExited + ? $"The test host exited with code {process.ExitCode} before it started. Its output above says why." + : $"The test host did not start within {options.StartTimeout.TotalSeconds:0} seconds."; + Kill(process); + process.Dispose(); + await pipe.DisposeAsync(); + cancellationToken.ThrowIfCancellationRequested(); + throw new TestHostException(message); + } + + /// Discovers the assembly's tests whose full name contains . + /// The assembly could not be loaded, or the host exited. + public async Task DiscoverAsync(string? filter, CancellationToken cancellationToken = default) + { + var reply = await RequestAsync(new TestHostMessage { Type = TestHostMessageTypes.Discover, Name = filter }, cancellationToken); + if (reply.Failure is { } failure) + throw new TestHostException(failure.Message); + + return new TestHostDiscovery( + reply.AssemblyName ?? Path.GetFileNameWithoutExtension(AssemblyPath), + reply.Tests ?? []); + } + + public Task InitializeAsync(CancellationToken cancellationToken = default) + { + lock (_initializeLock) + return _initialize ??= InitializeOnceAsync(); + } + + private async Task InitializeOnceAsync() + { + try + { + var reply = await RequestAsync(new TestHostMessage { Type = TestHostMessageTypes.Initialize }, CancellationToken.None); + return reply.Failure; + } + catch (TestHostException ex) + { + return new TestHostFailure(ex.Message, null); + } + } + + public async Task RunAsync(string testId, string fullName, CancellationToken cancellationToken = default) + { + if (_exitMessage is { } exited) + return TestCaseReport.NotRun(fullName, $"{exited} The test did not run."); + + var id = Interlocked.Increment(ref _nextId); + + // Stopping a run cancels the test's TestContext token in the host; the test runs on to its end. + using var cancel = cancellationToken.Register(() => + _connection.Post(new TestHostMessage { Type = TestHostMessageTypes.Cancel, Id = id })); + + try + { + var reply = await RequestAsync(new TestHostMessage { Type = TestHostMessageTypes.Run, Id = id, TestId = testId, Name = fullName }, CancellationToken.None); + return reply.Report ?? TestCaseReport.NotRun(fullName, "The test host sent no result."); + } + catch (TestHostException ex) + { + return TestCaseReport.NotRun(fullName, $"{ex.Message} The test was running at the time."); + } + } + + public async Task CleanupAsync(CancellationToken cancellationToken = default) + { + if (_exitMessage is not null) + return; + + try + { + await RequestAsync(new TestHostMessage { Type = TestHostMessageTypes.Cleanup }, cancellationToken); + } + catch (TestHostException) + { + // The host exited; the tests it was running have already said so. + } + } + + /// + /// Closes the pipe, which ends the test host, and waits briefly for it to exit before ending it + /// and the processes it started, such as a browser a test left open. + /// + public async ValueTask DisposeAsync() + { + _disposing = true; + await _connection.DisposeAsync(); + + try + { + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + Kill(_process); + } + + try { await _readLoop; } + catch { /* the loop reports through _exitMessage */ } + + _process.Dispose(); + } + + private async Task RequestAsync(TestHostMessage request, CancellationToken cancellationToken) + { + if (request.Id == 0) + request.Id = Interlocked.Increment(ref _nextId); + + var reply = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pending[request.Id] = reply; + + // The read loop sets the exit message before it fails what is pending, so a request added + // after it has looked is caught here. + if (_exitMessage is { } exited) + { + _pending.TryRemove(request.Id, out _); + throw new TestHostException(exited); + } + + _connection.Post(request); + return await reply.Task.WaitAsync(cancellationToken); + } + + private async Task ReadLoopAsync() + { + try + { + while (await _connection.ReadAsync() is { } message) + Dispatch(message); + } + catch (Exception ex) when (!_disposing) + { + _options.OnLog?.Invoke($"Lost the connection to the test host for {FileName}: {ex.Message}"); + } + + _exitMessage = _disposing + ? $"The test host for {FileName} has been shut down." + : await DescribeExitAsync(); + + foreach (var id in _pending.Keys) + { + if (_pending.TryRemove(id, out var pending)) + pending.TrySetException(new TestHostException(_exitMessage)); + } + } + + private async Task DescribeExitAsync() + { + try + { + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)); + return $"The test host for {FileName} exited unexpectedly with code {_process.ExitCode}."; + } + catch (TimeoutException) + { + return $"The test host for {FileName} closed its connection unexpectedly."; + } + } + + private void Dispatch(TestHostMessage message) + { + try + { + switch (message.Type) + { + case TestHostMessageTypes.Log: + if (message.Text is not null) + _options.OnLog?.Invoke(message.Text); + break; + + case TestHostMessageTypes.Frame: + if (message.Text is not null) + _options.OnFrame?.Invoke(message.Text); + break; + + case TestHostMessageTypes.Timeline: + if (message.Entry is not null) + _options.OnTimelineEntry?.Invoke(message.Entry); + break; + + case TestHostMessageTypes.Step: + _ = ContinueStepAsync(message); + break; + + default: + if (_pending.TryRemove(message.Id, out var pending)) + pending.TrySetResult(message); + break; + } + } + catch (Exception ex) + { + _options.OnLog?.Invoke($"Could not handle a {message.Type} message from the test host for {FileName}: {ex.Message}"); + } + } + + private async Task ContinueStepAsync(TestHostMessage step) + { + try + { + if (_options.OnStep is { } onStep) + await onStep(step.Text ?? "", step.Selector); + } + catch + { + // A step the runner could not hold still goes ahead. + } + + _connection.Post(new TestHostMessage { Type = TestHostMessageTypes.StepContinue, Id = step.Id }); + } + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Already gone. + } + } +} + +/// Builds the command line that starts a test host. +internal static class TestHostLauncher +{ + /// + /// A pipe name short enough for the socket path it becomes on Linux and macOS, which is the + /// temporary directory plus a prefix and is limited to about a hundred characters. + /// + public static string NewPipeName() => $"motus-{Guid.NewGuid():N}"[..22]; + + /// + /// dotnet exec of Motus.Runner.dll with the test assembly's own + /// .runtimeconfig.json and .deps.json, so that the test resolves its + /// dependencies, frameworks and base directory as it does under dotnet test. + /// Standard output and error are the runner's own. The process starts in the runner's current + /// directory and moves to the test assembly's folder once it has read its settings. + /// + public static ProcessStartInfo CreateStartInfo(string pipeName, string assemblyPath, bool visual) + { + var hostPath = typeof(TestHostServer).Assembly.Location; + var stem = Path.Combine(Path.GetDirectoryName(assemblyPath)!, Path.GetFileNameWithoutExtension(assemblyPath)); + var runtimeConfig = stem + ".runtimeconfig.json"; + var deps = stem + ".deps.json"; + + var startInfo = new ProcessStartInfo(FindDotnet()) + { + UseShellExecute = false, + WorkingDirectory = Environment.CurrentDirectory, + }; + + startInfo.ArgumentList.Add("exec"); + + // A class library built without a runtimeconfig runs on the runner's. + if (!File.Exists(runtimeConfig)) + runtimeConfig = Path.ChangeExtension(hostPath, ".runtimeconfig.json"); + if (File.Exists(runtimeConfig)) + { + startInfo.ArgumentList.Add("--runtimeconfig"); + startInfo.ArgumentList.Add(runtimeConfig); + } + + if (File.Exists(deps)) + { + startInfo.ArgumentList.Add("--depsfile"); + startInfo.ArgumentList.Add(deps); + } + + // Motus.Runner.dll was built for one .NET version and cannot load on an older one. A test + // built for an older one runs on the newest installed instead, as it did inside the runner. + if (NeedsNewerRuntime(runtimeConfig)) + { + startInfo.ArgumentList.Add("--roll-forward"); + startInfo.ArgumentList.Add("LatestMajor"); + } + + startInfo.ArgumentList.Add(hostPath); + startInfo.ArgumentList.Add(TestHostServer.Command); + startInfo.ArgumentList.Add(pipeName); + startInfo.ArgumentList.Add(assemblyPath); + if (visual) + startInfo.ArgumentList.Add(TestHostServer.VisualOption); + + return startInfo; + } + + /// + /// True when the runtimeconfig targets an older .NET major version than Motus.Runner.dll + /// was built for, or does not say which it targets. + /// + internal static bool NeedsNewerRuntime(string runtimeConfigPath) + { + var hostMajor = MajorVersionOf(typeof(TestHostServer).Assembly.GetCustomAttribute()?.FrameworkName); + if (hostMajor is null) + return false; + + try + { + var tfm = JsonNode.Parse(File.ReadAllText(runtimeConfigPath))?["runtimeOptions"]?["tfm"]?.GetValue(); + return MajorVersionOfTfm(tfm) is not { } testMajor || testMajor < hostMajor; + } + catch + { + return true; + } + } + + private static int? MajorVersionOfTfm(string? tfm) + { + // "net8.0", "net10.0-windows" + if (tfm is null || !tfm.StartsWith("net", StringComparison.OrdinalIgnoreCase)) + return null; + + var version = tfm[3..]; + var platform = version.IndexOf('-'); + if (platform >= 0) + version = version[..platform]; + return Version.TryParse(version, out var parsed) ? parsed.Major : null; + } + + private static int? MajorVersionOf(string? frameworkName) + { + // ".NETCoreApp,Version=v10.0" + var marker = frameworkName?.IndexOf("Version=v", StringComparison.Ordinal) ?? -1; + if (marker < 0) + return null; + + var version = frameworkName![(marker + "Version=v".Length)..]; + return Version.TryParse(version, out var parsed) ? parsed.Major : null; + } + + /// + /// The dotnet executable that runs this process, found from the runtime it loaded, + /// since a global tool runs from its own launcher rather than from dotnet. + /// + internal static string FindDotnet() + { + var executable = OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet"; + + if (Environment.ProcessPath is { } processPath + && string.Equals(Path.GetFileName(processPath), executable, StringComparison.OrdinalIgnoreCase)) + { + return processPath; + } + + // /shared/Microsoft.NETCore.App//System.Private.CoreLib.dll + var runtimeDirectory = Path.GetDirectoryName(typeof(object).Assembly.Location); + if (!string.IsNullOrEmpty(runtimeDirectory)) + { + var root = Path.GetFullPath(Path.Combine(runtimeDirectory, "..", "..", "..")); + var candidate = Path.Combine(root, executable); + if (File.Exists(candidate)) + return candidate; + } + + foreach (var variable in (string[])["DOTNET_HOST_PATH", "DOTNET_ROOT"]) + { + var value = Environment.GetEnvironmentVariable(variable); + if (string.IsNullOrEmpty(value)) + continue; + + var candidate = File.Exists(value) && !Directory.Exists(value) ? value : Path.Combine(value, executable); + if (File.Exists(candidate)) + return candidate; + } + + return executable; + } +} diff --git a/src/Motus.Runner/Services/Execution/TestHostProtocol.cs b/src/Motus.Runner/Services/Execution/TestHostProtocol.cs new file mode 100644 index 0000000..e5fd0e5 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestHostProtocol.cs @@ -0,0 +1,210 @@ +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Channels; +using Motus.Runner.Services.Timeline; + +namespace Motus.Runner.Services.Execution; + +/// The kinds of a runner and its test host exchange. +internal static class TestHostMessageTypes +{ + // Runner to test host. Each request carries an id that its reply repeats. + public const string Discover = "discover"; + public const string Initialize = "initialize"; + public const string Run = "run"; + public const string Cancel = "cancel"; + public const string Cleanup = "cleanup"; + public const string StepContinue = "step-continue"; + + // Test host to runner: replies. + public const string Discovered = "discovered"; + public const string Initialized = "initialized"; + public const string Result = "result"; + public const string CleanedUp = "cleaned-up"; + + // Test host to runner: notices, with no reply, except Step, which waits for StepContinue. + public const string Log = "log"; + public const string Frame = "frame"; + public const string Timeline = "timeline"; + public const string Step = "step"; +} + +/// +/// One message between a runner and a test host process. Only the members its +/// uses are set. +/// +internal sealed class TestHostMessage +{ + public string Type { get; set; } = ""; + + /// Pairs a request with its reply, and a step notice with its continue. + public int Id { get; set; } + + /// The test case a run names, or the name filter for discovery. + public string? Name { get; set; } + + /// The id of the test case a run asks for, which is looked up before . + public string? TestId { get; set; } + + /// A log line, a screencast frame (base64 JPEG), or the action a step paused before. + public string? Text { get; set; } + + /// The selector a paused step acts on. + public string? Selector { get; set; } + + public string? AssemblyName { get; set; } + + public List? Tests { get; set; } + + public TestCaseReport? Report { get; set; } + + public TestHostFailure? Failure { get; set; } + + public TimelineEntryMessage? Entry { get; set; } +} + +/// +/// A on its way from the test host to the visual runner, with its +/// performance metrics as JSON, since the two may run on different versions of Motus. +/// +internal sealed record TimelineEntryMessage( + DateTime Timestamp, + string ActionType, + string? Selector, + TimeSpan Duration, + byte[]? ScreenshotBefore, + byte[]? ScreenshotAfter, + bool HasError, + string? ErrorMessage, + List NetworkRequests, + List ConsoleMessages, + JsonElement? PerformanceMetrics) +{ + /// Made in the test host. Leaves out performance metrics it cannot write. + public static TimelineEntryMessage From(TimelineEntry entry) + { + JsonElement? metrics = null; + MotusInterop.Guard("timeline performance metrics", () => metrics = MetricsOf(entry)); + return new TimelineEntryMessage( + entry.Timestamp, entry.ActionType, entry.Selector, entry.Duration, entry.ScreenshotBefore, entry.ScreenshotAfter, + entry.HasError, entry.ErrorMessage, [.. entry.NetworkRequests], [.. entry.ConsoleMessages], metrics); + } + + /// Read in the runner, numbered and named for the test it belongs to. + public TimelineEntry ToEntry(int index, string? testName) + { + Abstractions.PerformanceMetrics? metrics = null; + if (PerformanceMetrics is { ValueKind: JsonValueKind.Object } json) + { + try { metrics = json.Deserialize(TestHostJson.Options); } + catch (JsonException) { /* shown without metrics */ } + } + + return new TimelineEntry(index, Timestamp, ActionType, Selector, Duration, ScreenshotBefore, ScreenshotAfter, + HasError, ErrorMessage, NetworkRequests, ConsoleMessages, testName, metrics); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static JsonElement? MetricsOf(TimelineEntry entry) + => entry.PerformanceMetrics is { } metrics ? JsonSerializer.SerializeToElement(metrics, TestHostJson.Options) : null; +} + +/// The JSON settings the runner and the test host share. +internal static class TestHostJson +{ + public static readonly JsonSerializerOptions Options = new() + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + }; +} + +/// +/// Reads and writes s on a stream, one JSON object per line. Writes +/// are queued and sent in the order they were posted, so a notice posted before a reply arrives +/// before it. +/// +internal sealed class TestHostConnection : IAsyncDisposable +{ + private static readonly UTF8Encoding s_utf8 = new(encoderShouldEmitUTF8Identifier: false); + + private readonly Stream _stream; + private readonly StreamReader _reader; + private readonly Channel _outgoing = Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true }); + private readonly Task _writer; + + public TestHostConnection(Stream stream) + { + _stream = stream; + _reader = new StreamReader(stream, s_utf8, detectEncodingFromByteOrderMarks: false, bufferSize: 64 * 1024, leaveOpen: true); + _writer = Task.Run(WriteLoopAsync); + } + + /// Queues to be sent. Does nothing once the connection has closed. + public void Post(TestHostMessage message) => _outgoing.Writer.TryWrite(message); + + /// The next message, or null when the other side has closed the connection. + public async Task ReadAsync(CancellationToken cancellationToken = default) + { + while (true) + { + string? line; + try + { + line = await _reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + return null; + } + catch (ObjectDisposedException) + { + return null; + } + + if (line is null) + return null; + if (line.Length == 0) + continue; + + return JsonSerializer.Deserialize(line, TestHostJson.Options); + } + } + + private async Task WriteLoopAsync() + { + try + { + await foreach (var message in _outgoing.Reader.ReadAllAsync().ConfigureAwait(false)) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes(message, TestHostJson.Options); + await _stream.WriteAsync(bytes).ConfigureAwait(false); + await _stream.WriteAsync("\n"u8.ToArray()).ConfigureAwait(false); + await _stream.FlushAsync().ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + // The other side has gone; the read side reports it. + } + finally + { + _outgoing.Writer.TryComplete(); + } + } + + /// Sends what is queued, then closes the stream. + public async ValueTask DisposeAsync() + { + _outgoing.Writer.TryComplete(); + try { await _writer.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } + catch { /* the stream is closed below either way */ } + + _reader.Dispose(); + await _stream.DisposeAsync().ConfigureAwait(false); + } +} diff --git a/src/Motus.Runner/Services/Execution/TestHostServer.cs b/src/Motus.Runner/Services/Execution/TestHostServer.cs new file mode 100644 index 0000000..568a880 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestHostServer.cs @@ -0,0 +1,360 @@ +using System.Collections.Concurrent; +using System.IO.Pipes; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.Loader; +using Motus.Abstractions; +using Motus.Runner.Services.Timeline; + +namespace Motus.Runner.Services.Execution; + +/// +/// The test host: a process of its own for one test assembly, started by +/// from Motus.Runner.dll with the test assembly's own .deps.json and +/// .runtimeconfig.json, the way dotnet test starts its test host. The test therefore +/// gets its own versions of everything it depends on, the framework assemblies it was built +/// against, and its own output folder as . It serves +/// discovery and test runs to the runner over a named pipe. +/// +internal static class TestHostServer +{ + /// The first argument that makes Motus.Runner.dll a test host rather than the visual runner. + internal const string Command = "--motus-test-host"; + + /// Asks the test host to stream the browser to the visual runner and record its timeline. + internal const string VisualOption = "--visual"; + + /// True when start a test host: the command, a pipe name and a test assembly. + public static bool IsHostCommand(string[] args) => args.Length >= 3 && args[0] == Command; + + /// + /// Makes the test assembly's folder the base directory and the current directory, as they are + /// under dotnet test, and lets the runner's own assemblies load beside the test's. Called + /// before anything uses a Motus type, since those resolve through it when the test was built + /// against another Motus version or does not reference Motus at all. + /// + public static void Prepare(string[] args) + { + var assemblyPath = Path.GetFullPath(args[2]); + var testDirectory = Path.GetDirectoryName(assemblyPath)!; + var runnerDirectory = Path.GetDirectoryName(typeof(TestHostServer).Assembly.Location)!; + + // The host already does this when the test has a .deps.json, which sets the base directory + // to its folder. A test without one still expects its own. + AppContext.SetData("APP_CONTEXT_BASE_DIRECTORY", testDirectory + Path.DirectorySeparatorChar); + + AssemblyLoadContext.Default.Resolving += (context, name) => + { + // A version the test's dependencies don't have, such as the runner asking for a newer + // Motus than the test references: the test's own copy is the one to use. + if (name.Version is not null && name.Name is not null) + { + try { return context.LoadFromAssemblyName(new AssemblyName(name.Name)); } + catch (Exception ex) when (ex is FileNotFoundException or FileLoadException) { } + } + + // Not one of the test's dependencies: the runner's own, or a test without a .deps.json. + foreach (var directory in (string[])[testDirectory, runnerDirectory]) + { + var path = Path.Combine(directory, name.Name + ".dll"); + if (File.Exists(path)) + return context.LoadFromAssemblyPath(path); + } + + return null; + }; + + // The process starts in the folder motus run was started in. motus.config.json is read from + // there, where the runner reads it, so that both see the same settings; it is read once per + // process, so reading it now keeps it. Then the tests get their own folder, where + // WebApplicationFactory looks for the app's manifest and the tests' relative paths resolve. + try + { + MotusInterop.LoadConfig(); + } + catch + { + // Motus reads its settings when it first needs them, from the folder below. + } + + Environment.CurrentDirectory = testDirectory; + } + + /// Serves the runner until it closes the pipe. Returns the process exit code. + public static async Task RunAsync(string[] args) + { + var pipeName = args[1]; + var assemblyPath = Path.GetFullPath(args[2]); + var visual = args.Skip(3).Contains(VisualOption); + + var pipe = new NamedPipeClientStream(".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous); + try + { + await pipe.ConnectAsync((int)TimeSpan.FromSeconds(30).TotalMilliseconds); + } + catch (Exception ex) + { + Console.Error.WriteLine($"The Motus test host could not reach the runner: {ex.Message}"); + await pipe.DisposeAsync(); + return 1; + } + + await using var connection = new TestHostConnection(pipe); + await new Session(connection, assemblyPath, visual).ServeAsync(); + return 0; + } + + private sealed class Session(TestHostConnection connection, string assemblyPath, bool visual) + { + private readonly ConcurrentDictionary _runs = new(); + private readonly ConcurrentDictionary _steps = new(); + private InProcessTestHost? _host; + private int _nextStepId; + + public async Task ServeAsync() + { + MotusInterop.Warn = Log; + + if (visual) + InstallVisualHooks(); + + while (await connection.ReadAsync() is { } message) + { + switch (message.Type) + { + case TestHostMessageTypes.Discover: + // Before anything else, and nothing else is sent until it has replied. + Reply(message, Discover(message.Name)); + break; + + case TestHostMessageTypes.Initialize: + _ = Task.Run(async () => Reply(message, new TestHostMessage + { + Type = TestHostMessageTypes.Initialized, + Failure = await Host.InitializeAsync(), + })); + break; + + case TestHostMessageTypes.Run: + var cancellation = new CancellationTokenSource(); + _runs[message.Id] = cancellation; + _ = Task.Run(async () => + { + TestCaseReport report; + try + { + report = await Host.RunAsync(message.TestId ?? message.Name ?? "", message.Name ?? "", cancellation.Token); + } + catch (Exception ex) + { + report = TestCaseReport.NotRun(message.Name ?? "", ex.Message); + } + finally + { + _runs.TryRemove(message.Id, out _); + } + + Reply(message, new TestHostMessage { Type = TestHostMessageTypes.Result, Report = report }); + }); + break; + + case TestHostMessageTypes.Cancel: + if (_runs.TryGetValue(message.Id, out var running)) + running.Cancel(); + break; + + case TestHostMessageTypes.Cleanup: + _ = Task.Run(async () => + { + await Host.CleanupAsync(); + Reply(message, new TestHostMessage { Type = TestHostMessageTypes.CleanedUp }); + }); + break; + + case TestHostMessageTypes.StepContinue: + if (_steps.TryRemove(message.Id, out var step)) + step.TrySetResult(); + break; + } + } + + // The runner has closed the pipe, whether it finished or went away. + } + + private InProcessTestHost Host => _host ?? throw new InvalidOperationException("No test assembly has been discovered."); + + private TestHostMessage Discover(string? filter) + { + try + { + var assembly = LoadTestAssembly(); + WarnAboutMotusVersion(); + var cases = TestCaseDiscovery.Discover(assembly, filter, Log); + _host = new InProcessTestHost(assembly, cases, Log); + + return new TestHostMessage + { + Type = TestHostMessageTypes.Discovered, + AssemblyName = assembly.GetName().Name, + Tests = cases.Select(c => c.ToInfo()).ToList(), + }; + } + catch (Exception ex) + { + var inner = ex is TargetInvocationException { InnerException: { } e } ? e : ex; + return new TestHostMessage + { + Type = TestHostMessageTypes.Discovered, + Failure = new TestHostFailure(inner.Message, inner.StackTrace), + }; + } + } + + private Assembly LoadTestAssembly() + { + // With the test's .deps.json the assembly is one of the application's own, and loading + // it by name gives the copy everything else binds to. Without one, it is loaded by path. + try + { + return AssemblyLoadContext.Default.LoadFromAssemblyName(AssemblyName.GetAssemblyName(assemblyPath)); + } + catch (FileNotFoundException) + { + return AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); + } + } + + /// + /// The tests run on the Motus their project references, which the runner reaches into for + /// its results. Says so when that is not the version this runner was built with. + /// + private void WarnAboutMotusVersion() + { + var expected = typeof(TestHostServer).Assembly.GetReferencedAssemblies().FirstOrDefault(a => a.Name == "Motus")?.Version; + var actual = MotusInterop.LoadedMotusVersion(); + if (expected is null || actual is null || expected == actual) + return; + + Log($"{Path.GetFileName(assemblyPath)} references Motus {actual.ToString(3)} and this motus is {expected.ToString(3)}. " + + "The tests run on the Motus they reference. If results or reports are missing, use the same version of both."); + } + + private void InstallVisualHooks() + { + try + { + MotusInterop.Guard("the visual runner's browser view and timeline", InstallVisualHooksCore); + } + catch (Exception ex) + { + Log($"The visual runner cannot show the browser for {Path.GetFileName(assemblyPath)}: {ex.Message}"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void InstallVisualHooksCore() + { + // Compile what the hooks will run now, so that a Motus without something they use turns + // the browser view off here rather than failing inside a test's browser action. + PrepareMethods(typeof(ScreencastService), except: nameof(ScreencastService.HighlightElementAsync)); + PrepareMethods(typeof(TimelineRecorderHook), except: null); + + var screencast = new ScreencastService(); + screencast.FrameReceived += frame => connection.Post(new TestHostMessage { Type = TestHostMessageTypes.Frame, Text = frame }); + var timeline = new RemoteTimeline(entry => connection.Post(new TestHostMessage + { + Type = TestHostMessageTypes.Timeline, + Entry = TimelineEntryMessage.From(entry), + })); + var stepDebug = new RemoteStepDebug(StepAsync); + + // The pages the tests open, the way the visual runner watches them in its own process. + BrowserContext.GlobalPageCreated = page => + { + _ = Task.Run(async () => + { + try { await screencast.AttachPageAsync(page); } + catch { /* best-effort */ } + }); + + page.Context.GetPluginContext().RegisterLifecycleHook(new TimelineRecorderHook(timeline, stepDebug)); + }; + } + + /// + /// JIT-compiles the methods of , including the state machines and + /// closures of its async methods and lambdas, apart from those of . + /// + private static void PrepareMethods(Type type, string? except) + { + const BindingFlags all = BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly; + var types = new[] { type }.Concat(type.GetNestedTypes(BindingFlags.NonPublic | BindingFlags.Public)) + .Where(t => except is null || !t.Name.Contains($"<{except}>", StringComparison.Ordinal) && !t.IsGenericTypeDefinition); + + foreach (var candidate in types) + { + foreach (var method in candidate.GetMethods(all).Cast().Concat(candidate.GetConstructors(all))) + { + if (method.IsAbstract || method.ContainsGenericParameters || method.Name == except) + continue; + RuntimeHelpers.PrepareMethod(method.MethodHandle); + } + } + } + + private async Task StepAsync(string action, string? selector, CancellationToken cancellationToken) + { + var id = Interlocked.Increment(ref _nextStepId); + var step = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _steps[id] = step; + connection.Post(new TestHostMessage { Type = TestHostMessageTypes.Step, Id = id, Text = action, Selector = selector }); + await step.Task.WaitAsync(cancellationToken); + } + + private void Log(string message) => connection.Post(new TestHostMessage { Type = TestHostMessageTypes.Log, Text = message }); + + private void Reply(TestHostMessage request, TestHostMessage reply) + { + reply.Id = request.Id; + connection.Post(reply); + } + } + + /// + /// The timeline the recorder hook writes to inside the test host. Entries go to the visual + /// runner, which numbers them and names their test. + /// + private sealed class RemoteTimeline(Action send) : ITimelineService + { + public IReadOnlyList Entries => []; + public int? SelectedIndex => null; + public TimelineEntry? SelectedEntry => null; + public string? CurrentTestName { get; set; } + public string? SelectedTestName => null; + public event Action? TimelineChanged { add { } remove { } } + public event Action? TestSelected { add { } remove { } } + public void Clear() { } + public void AddEntry(TimelineEntry entry) => send(entry); + public void SelectEntry(int index) { } + public void ClearSelection() { } + public void SelectTest(string fullName) { } + } + + /// + /// Step debugging inside the test host: each action asks the visual runner, which holds it + /// while the user steps. + /// + private sealed class RemoteStepDebug(Func step) : IStepDebugService + { + public bool IsStepMode => false; + public bool IsPaused => false; + public string? PendingActionType => null; + public string? PendingSelector => null; + public event Action? StateChanged { add { } remove { } } + public void EnableStepMode() { } + public void DisableStepMode() { } + public void Advance() { } + public void Resume() { } + public Task WaitIfPausedAsync(string actionType, string? selector, CancellationToken ct) => step(actionType, selector, ct); + } +} diff --git a/src/Motus.Runner/Services/Models/DiscoveredTest.cs b/src/Motus.Runner/Services/Models/DiscoveredTest.cs index 4737ec4..c7a754a 100644 --- a/src/Motus.Runner/Services/Models/DiscoveredTest.cs +++ b/src/Motus.Runner/Services/Models/DiscoveredTest.cs @@ -2,6 +2,14 @@ namespace Motus.Runner.Services.Models; +/// +/// A test the visual runner lists. A test found by discovery runs in a test host process of its +/// assembly (at ), so , +/// and are null and it is known by +/// and . One built from reflection +/// metadata runs in this process. identifies a discovered test apart from +/// its name, which a data row's data source may choose differently each time it is read. +/// public sealed record DiscoveredTest( Type? TestClass, MethodInfo? TestMethod, @@ -12,4 +20,6 @@ public sealed record DiscoveredTest( string? CodeBase = null, IReadOnlyList? Categories = null, string? ClassFullName = null, - object?[]? Arguments = null); + object?[]? Arguments = null, + string? MethodName = null, + string? TestId = null); diff --git a/src/Motus.Runner/Services/ScreencastService.cs b/src/Motus.Runner/Services/ScreencastService.cs index 7227f40..90c4115 100644 --- a/src/Motus.Runner/Services/ScreencastService.cs +++ b/src/Motus.Runner/Services/ScreencastService.cs @@ -9,8 +9,10 @@ internal sealed class ScreencastService : IScreencastService, IAsyncDisposable private CancellationTokenSource? _pumpCts; private Task? _pumpTask; + private volatile bool _showingRemoteFrames; + public string? LatestFrameBase64 { get; private set; } - public bool IsStreaming => _pumpTask is { IsCompleted: false }; + public bool IsStreaming => _pumpTask is { IsCompleted: false } || _showingRemoteFrames; public ElementHighlight? CurrentHighlight { get; private set; } public event Action? FrameReceived; @@ -56,6 +58,20 @@ private void OnPageClose(object? sender, EventArgs e) _pumpCts?.Cancel(); } + /// + /// Shows a frame of a page in another process: the test host streams the pages of the tests it + /// runs, which this process cannot attach to. + /// + internal void ShowRemoteFrame(string base64) + { + _showingRemoteFrames = true; + LatestFrameBase64 = base64; + FrameReceived?.Invoke(base64); + } + + /// Ends what started, when the test host has gone. + internal void EndRemoteFrames() => _showingRemoteFrames = false; + public async Task HighlightElementAsync(string? objectId, CancellationToken ct = default) { if (objectId is null || _page is null) diff --git a/src/Motus.Runner/Services/TestDiscovery.cs b/src/Motus.Runner/Services/TestDiscovery.cs index d77ccdb..781c61e 100644 --- a/src/Motus.Runner/Services/TestDiscovery.cs +++ b/src/Motus.Runner/Services/TestDiscovery.cs @@ -1,123 +1,56 @@ -using System.Reflection; using Motus.Runner.Services.Execution; using Motus.Runner.Services.Models; namespace Motus.Runner.Services; +/// +/// Finds the tests in test assemblies for the visual runner. Each assembly is loaded by a test host +/// process of its own, started from the assembly's output folder with its own dependencies, which +/// exits once it has listed the tests. Runs start a fresh one (see ), +/// so a rebuilt assembly is picked up by the next run. +/// public sealed class TestDiscovery(ILogger logger) { - private static readonly HashSet ClassAttributes = new(StringComparer.Ordinal) - { - "TestClassAttribute", - "TestFixtureAttribute", - }; - - private static readonly HashSet MethodAttributes = new(StringComparer.Ordinal) - { - "TestMethodAttribute", - "TestAttribute", - "FactAttribute", - }; - - // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply. - private static readonly HashSet ExcludedMethodAttributes = new(StringComparer.Ordinal) - { - "TheoryAttribute", - }; - - private static readonly HashSet IgnoreAttributes = new(StringComparer.Ordinal) - { - "IgnoreAttribute", // MSTest + NUnit - }; - - public List Discover(string[] assemblyPaths, string? filter) + /// + /// The tests in whose full name contains , + /// ignoring case. An assembly that cannot be loaded is logged and skipped. + /// + public async Task> DiscoverAsync(string[] assemblyPaths, string? filter, CancellationToken cancellationToken = default) { var tests = new List(); foreach (var path in assemblyPaths) { var fullPath = Path.GetFullPath(path); - Assembly assembly; try { - assembly = Assembly.LoadFrom(fullPath); - } - catch (Exception ex) - { - logger.LogError("Failed to load assembly {Path}: {Message}", fullPath, ex.Message); - continue; - } - - var assemblyName = assembly.GetName().Name ?? Path.GetFileNameWithoutExtension(fullPath); - - foreach (var type in assembly.GetExportedTypes()) - { - if (!IsTestClass(type)) - continue; + await using var host = await TestHostClient.StartAsync( + fullPath, + new TestHostOptions { OnLog = message => logger.LogWarning("{Message}", message) }, + cancellationToken); + var discovery = await host.DiscoverAsync(filter, cancellationToken); - var (classIgnored, classReason) = GetIgnoreInfo(type.GetCustomAttributes(true)); - - foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance)) + foreach (var test in discovery.Tests) { - if (!IsTestMethod(method)) - continue; - - var (methodIgnored, methodReason) = GetIgnoreInfo(method.GetCustomAttributes(true)); - - foreach (var testCase in TestCaseExpander.Expand(type, method, message => logger.LogWarning("{Message}", message))) - { - if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) - continue; - - var isIgnored = classIgnored || methodIgnored || testCase.IgnoreReason is not null; - var reason = classIgnored ? classReason : methodIgnored ? methodReason : testCase.IgnoreReason; - tests.Add(new DiscoveredTest(type, method, testCase.FullName, assemblyName, isIgnored, reason, - Arguments: testCase.Arguments)); - } + tests.Add(new DiscoveredTest( + TestClass: null, + TestMethod: null, + test.FullName, + discovery.AssemblyName, + test.IsIgnored, + test.IgnoreReason, + CodeBase: fullPath, + ClassFullName: test.ClassName, + MethodName: test.MethodName, + TestId: test.Id)); } } + catch (TestHostException ex) + { + logger.LogError("Failed to load assembly {Path}: {Message}", fullPath, ex.Message); + } } return tests; } - - private static bool IsTestClass(Type type) - { - if (type.IsAbstract || !type.IsClass) - return false; - - return type.GetCustomAttributes(true) - .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes)); - } - - private static bool IsTestMethod(MethodInfo method) - { - return method.GetCustomAttributes(true) - .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes) - && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes)); - } - - private static (bool IsIgnored, string? Reason) GetIgnoreInfo(object[] attrs) - { - // MSTest [Ignore] / [Ignore("reason")] / NUnit [Ignore("reason")] - var ignoreAttr = attrs.FirstOrDefault(a => IgnoreAttributes.Contains(a.GetType().Name)); - if (ignoreAttr is not null) - { - // MSTest uses IgnoreMessage; NUnit uses Reason - var msg = ignoreAttr.GetType().GetProperty("IgnoreMessage")?.GetValue(ignoreAttr) as string - ?? ignoreAttr.GetType().GetProperty("Reason")?.GetValue(ignoreAttr) as string; - return (true, msg); - } - - // xUnit [Fact(Skip = "reason")] - var factAttr = attrs.FirstOrDefault(a => a.GetType().Name == "FactAttribute"); - if (factAttr is not null) - { - var skip = factAttr.GetType().GetProperty("Skip")?.GetValue(factAttr) as string; - if (skip is not null) - return (true, skip); - } - - return (false, null); - } } diff --git a/src/Motus.Runner/Services/TestExecutionService.cs b/src/Motus.Runner/Services/TestExecutionService.cs index 944c244..ec117f7 100644 --- a/src/Motus.Runner/Services/TestExecutionService.cs +++ b/src/Motus.Runner/Services/TestExecutionService.cs @@ -1,26 +1,28 @@ -using System.Collections.Concurrent; using System.Diagnostics; -using System.Reflection; -using Motus; using Motus.Abstractions; using Motus.Runner.Services.Coverage; using Motus.Runner.Services.Execution; using Motus.Runner.Services.Models; +using Motus.Runner.Services.Timeline; namespace Motus.Runner.Services; -public sealed class TestExecutionService(ILogger logger, ICoverageService? coverageService = null) +/// +/// Runs tests for the visual runner. Each run starts a test host process for each test assembly +/// it covers, which streams the browser and the timeline back here, and ends it when the run is +/// over. Tests built from reflection metadata rather than discovered run in this process. +/// +public sealed class TestExecutionService( + ILogger logger, + ICoverageService? coverageService = null, + IScreencastService? screencast = null, + ITimelineService? timeline = null, + IStepDebugService? stepDebug = null) { // Browser automation tests must run sequentially to avoid contention // on the shared browser process and screenshot capture pipeline private const int MaxWorkers = 1; - // Track which assemblies have had [AssemblyInitialize] run - private readonly ConcurrentDictionary _initializedAssemblies = new(); - - // Track which classes have had [ClassInitialize] / [OneTimeSetUp] run in this batch - private readonly ConcurrentDictionary _initializedClasses = new(); - public async Task ExecuteAsync( List tests, Action onStateChanged, @@ -29,46 +31,122 @@ public async Task ExecuteAsync( { foreach (var test in tests) { - if (test.TestClass is null || test.TestMethod is null) + if ((test.TestClass is null || test.TestMethod is null) && string.IsNullOrEmpty(test.CodeBase)) throw new InvalidOperationException( $"Test '{test.FullName}' cannot be executed: missing reflection metadata (TRX-loaded tests are read-only)."); } - var suiteName = tests.Count > 0 - ? tests[0].TestClass!.Assembly.GetName().Name ?? "Motus Tests" - : "Motus Tests"; + var suiteName = tests.Count > 0 ? tests[0].AssemblyName : "Motus Tests"; if (reporters is not null) await reporters.FireOnTestRunStartAsync(new TestSuiteInfo(suiteName, tests.Count)); - // Run [AssemblyInitialize] for each assembly represented in this batch - var assemblies = tests.Select(t => t.TestClass!.Assembly).Distinct().ToList(); - foreach (var assembly in assemblies) + var hosts = new Dictionary(StringComparer.Ordinal); + var runnable = new List(); + var notRun = 0; + try { - if (_initializedAssemblies.TryAdd(assembly, true)) + // Start a test host for each assembly in this batch and run its [AssemblyInitialize]. An + // assembly that can't start or initialize fails its own tests; the others still run. + foreach (var group in tests.GroupBy(AssemblyKeyOf)) { + var first = group.First(); + ITestAssemblyHost host; try { - await RunAssemblyInitializeAsync(assembly); + host = await CreateHostAsync(first, group, ct); + } + catch (TestHostException ex) + { + logger.LogError("Could not run the tests in {Assembly}: {Message}", first.AssemblyName, ex.Message); + notRun += await FailAllAsync(group, $"Could not run the tests in {first.AssemblyName}: {ex.Message}", null, + suiteName, onStateChanged, reporters); + continue; } - catch (Exception ex) + catch (OperationCanceledException) { - logger.LogError(ex, "AssemblyInitialize failed for {Assembly}", assembly.GetName().Name); - // Fail all tests in this assembly - foreach (var test in tests.Where(t => t.TestClass!.Assembly == assembly)) - { - onStateChanged(new TestNodeState(test.FullName, TestStatus.Failed, null, - $"AssemblyInitialize failed: {ex.InnerException?.Message ?? ex.Message}", - ex.InnerException?.StackTrace ?? ex.StackTrace)); - } + // Stopped while a test host was starting. + foreach (var test in tests) + onStateChanged(new TestNodeState(test.FullName, TestStatus.Skipped, null, "Cancelled", null)); return; } + + hosts[group.Key] = host; + + var failure = await host.InitializeAsync(ct); + if (failure is not null) + { + logger.LogError("AssemblyInitialize failed for {Assembly}: {Message}", first.AssemblyName, failure.Message); + notRun += await FailAllAsync(group, $"AssemblyInitialize failed: {failure.Message}", failure.StackTrace, + suiteName, onStateChanged, reporters); + hosts.Remove(group.Key); + await host.DisposeAsync(); + continue; + } + + runnable.AddRange(group); } + + await RunTestsAsync(runnable, hosts, suiteName, onStateChanged, ct, reporters, notRun); + + // Run [ClassCleanup] / [OneTimeTearDown] for each class that ran a test, then + // [AssemblyCleanup] to close browsers + foreach (var host in hosts.Values) + await host.CleanupAsync(); + } + finally + { + foreach (var host in hosts.Values) + await host.DisposeAsync(); + + (screencast as ScreencastService)?.EndRemoteFrames(); } + } + /// Fails each of with . Returns how many failed. + private static async Task FailAllAsync( + IEnumerable tests, + string message, + string? stackTrace, + string suiteName, + Action onStateChanged, + ReporterCollection? reporters) + { + var count = 0; + foreach (var test in tests) + { + if (test.IsIgnored) + { + onStateChanged(new TestNodeState(test.FullName, TestStatus.Skipped, null, test.IgnoreReason ?? "Ignored", null)); + continue; + } + + onStateChanged(new TestNodeState(test.FullName, TestStatus.Failed, null, message, stackTrace)); + if (reporters is not null) + { + var info = new TestInfo(test.FullName, suiteName); + await reporters.FireOnTestStartAsync(info); + await reporters.FireOnTestEndAsync(info, new Abstractions.TestResult(test.FullName, false, 0, message, stackTrace)); + } + + count++; + } + + return count; + } + + private async Task RunTestsAsync( + List tests, + Dictionary hosts, + string suiteName, + Action onStateChanged, + CancellationToken ct, + ReporterCollection? reporters, + int alreadyFailed) + { var sw = reporters is not null ? Stopwatch.StartNew() : null; var semaphore = new SemaphoreSlim(MaxWorkers); - int passedCount = 0, failedCount = 0, skippedCount = 0; + int passedCount = 0, failedCount = alreadyFailed, skippedCount = 0; var collectedCoverage = new List(); var coverageLock = new object(); @@ -91,12 +169,13 @@ public async Task ExecuteAsync( if (reporters is not null) await reporters.FireOnTestStartAsync(new TestInfo(test.FullName, suiteName)); - CoverageSink.Begin(); - var result = await ExecuteTestAsync(test, ct); + var report = await hosts[AssemblyKeyOf(test)].RunAsync(test.TestId ?? test.FullName, test.FullName, ct); + var result = report.Passed + ? new TestNodeState(test.FullName, TestStatus.Passed, report.Duration, null, null) + : new TestNodeState(test.FullName, TestStatus.Failed, report.Duration, report.ErrorMessage, report.StackTrace); onStateChanged(result); - var coverage = CoverageSink.End(); - if (coverage is not null) + if (TestCaseReportExtras.Coverage(report) is { } coverage) { lock (coverageLock) collectedCoverage.Add(coverage); @@ -137,39 +216,6 @@ public async Task ExecuteAsync( logger.LogInformation("Test run cancelled"); } - // Run [ClassCleanup] / [OneTimeTearDown] for each class initialized in this batch - foreach (var type in _initializedClasses.Keys.ToList()) - { - if (_initializedClasses.TryRemove(type, out _)) - { - try - { - await RunStaticLifecycleMethodsAsync(type, "ClassCleanupAttribute"); - await RunStaticLifecycleMethodsAsync(type, "OneTimeTearDownAttribute"); - } - catch (Exception ex) - { - logger.LogWarning(ex, "ClassCleanup failed for {Class}", type.FullName); - } - } - } - - // Run [AssemblyCleanup] for each initialized assembly to close browsers - foreach (var assembly in assemblies) - { - if (_initializedAssemblies.TryRemove(assembly, out _)) - { - try - { - await RunAssemblyCleanupAsync(assembly); - } - catch (Exception ex) - { - logger.LogWarning(ex, "AssemblyCleanup failed for {Assembly}", assembly.GetName().Name); - } - } - } - if (reporters is not null) { sw!.Stop(); @@ -195,70 +241,43 @@ await reporters.FireOnTestRunEndAsync( } } - private static async Task RunAssemblyInitializeAsync(Assembly assembly) + /// Groups the tests of one assembly: by the assembly itself in this process, by its path otherwise. + private static string AssemblyKeyOf(DiscoveredTest test) + => test.TestClass is not null && test.TestMethod is not null + ? $"in-process:{test.TestClass.Assembly.FullName}" + : Path.GetFullPath(test.CodeBase!); + + private async Task CreateHostAsync(DiscoveredTest first, IEnumerable tests, CancellationToken ct) { - // [AssemblyInitialize] is a static method that takes TestContext - // Look across all types in the assembly for it - foreach (var type in assembly.GetExportedTypes()) + if (first.TestClass is not null && first.TestMethod is not null) { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyInitializeAttribute")); - - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); + var cases = tests.Select(t => new DiscoveredTestCase(t.TestClass!, t.TestMethod!, t.FullName, t.Arguments, t.IsIgnored, t.IgnoreReason, Quarantined: false, t.TestId)); + return new InProcessTestHost(first.TestClass.Assembly, cases, message => logger.LogWarning("{Message}", message)); } - } - private static async Task RunAssemblyCleanupAsync(Assembly assembly) - { - foreach (var type in assembly.GetExportedTypes()) + var remoteScreencast = screencast as ScreencastService; + var host = await TestHostClient.StartAsync(first.CodeBase!, new TestHostOptions { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyCleanupAttribute")); - - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); + Visual = true, + OnLog = message => logger.LogWarning("{Message}", message), + OnFrame = remoteScreencast is null ? null : remoteScreencast.ShowRemoteFrame, + OnTimelineEntry = timeline is null ? null : entry => + timeline.AddEntry(entry.ToEntry(timeline.Entries.Count, timeline.CurrentTestName)), + OnStep = stepDebug is null ? null : (action, selector) => stepDebug.WaitIfPausedAsync(action, selector, CancellationToken.None), + }, ct); + + // A fresh process discovers again. The run asks for each test by its id, which a data row keeps + // even when its data source names it differently this time, and one that is gone fails saying so. + try + { + await host.DiscoverAsync(filter: null, ct); + } + catch + { + await host.DisposeAsync(); + throw; } - } - - private static async Task RunStaticLifecycleMethodsAsync(Type type, string attributeName) - { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); - - foreach (var method in methods) - await TestCaseInvoker.InvokeStaticAsync(method, type); - } - - private async Task ExecuteTestAsync(DiscoveredTest test, CancellationToken ct) - { - var testClass = test.TestClass!; - var run = await TestCaseInvoker.RunAsync( - testClass, - test.TestMethod!, - test.Arguments, - DisplayNameOf(test), - beforeInstance: async () => - { - // Run [ClassInitialize] / [OneTimeSetUp] once per class, before its first test - if (_initializedClasses.TryAdd(testClass, true)) - { - await RunStaticLifecycleMethodsAsync(testClass, "ClassInitializeAttribute"); - await RunStaticLifecycleMethodsAsync(testClass, "OneTimeSetUpAttribute"); - } - }, - onCleanupFailure: ex => logger.LogWarning(ex, "TestCleanup failed for {Test}", test.FullName), - cancellationToken: ct); - - return run.Failure is null - ? new TestNodeState(test.FullName, TestStatus.Passed, run.Duration, null, null) - : new TestNodeState(test.FullName, TestStatus.Failed, run.Duration, run.Failure.Message, run.Failure.StackTrace); - } - /// The test case's name without its class, such as SignsIn ("owner"). - private static string DisplayNameOf(DiscoveredTest test) - { - var prefix = test.TestClass!.FullName + "."; - return test.FullName.StartsWith(prefix, StringComparison.Ordinal) ? test.FullName[prefix.Length..] : test.TestMethod!.Name; + return host; } } diff --git a/src/Motus.Runner/Services/TestSessionService.cs b/src/Motus.Runner/Services/TestSessionService.cs index 67f2eda..bb598b8 100644 --- a/src/Motus.Runner/Services/TestSessionService.cs +++ b/src/Motus.Runner/Services/TestSessionService.cs @@ -33,9 +33,9 @@ public TestSessionService(TestDiscovery discovery, TestExecutionService executor public event Action? StateChanged; public event Action? RunStarted; - public Task LoadAssembliesAsync(string[] paths, string? filter) + public async Task LoadAssembliesAsync(string[] paths, string? filter) { - var tests = _discovery.Discover(paths, filter); + var tests = await _discovery.DiscoverAsync(paths, filter); _discoveredTests = tests; _states.Clear(); @@ -47,7 +47,6 @@ public Task LoadAssembliesAsync(string[] paths, string? filter) } NotifyStateChanged(); - return Task.CompletedTask; } public void LoadFromTrxResults(IReadOnlyList tests, IReadOnlyDictionary states) @@ -144,7 +143,7 @@ public async Task RunClassAsync(string className, CancellationToken ct = default try { _timeline.Clear(); - var tests = _discoveredTests.Where(t => t.TestClass?.FullName == className).ToList(); + var tests = _discoveredTests.Where(t => (t.TestClass?.FullName ?? t.ClassFullName) == className).ToList(); _lastRunTests = new HashSet(tests.Select(t => t.FullName)); foreach (var test in tests) { diff --git a/src/Motus.Runner/Services/TrxRerunService.cs b/src/Motus.Runner/Services/TrxRerunService.cs index cb4b61f..310a994 100644 --- a/src/Motus.Runner/Services/TrxRerunService.cs +++ b/src/Motus.Runner/Services/TrxRerunService.cs @@ -36,7 +36,7 @@ public async Task RerunTestAsync(string fullName, CancellationToken ct = default DiscoveredTest? reflected; try { - var candidates = discovery.Discover([codeBase], fullName); + var candidates = await discovery.DiscoverAsync([codeBase], fullName, ct); reflected = candidates.FirstOrDefault(t => t.FullName == fullName); } catch (Exception ex) @@ -50,7 +50,7 @@ public async Task RerunTestAsync(string fullName, CancellationToken ct = default return; } - if (reflected?.TestClass is null || reflected.TestMethod is null) + if (reflected is null) { session.SetTestState(new TestNodeState( fullName, diff --git a/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/Motus.Fixtures.OldMotus.Tests.csproj b/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/Motus.Fixtures.OldMotus.Tests.csproj new file mode 100644 index 0000000..31ee613 --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/Motus.Fixtures.OldMotus.Tests.csproj @@ -0,0 +1,23 @@ + + + + + + net8.0;net10.0 + latest + enable + enable + false + + + + + + + + + + diff --git a/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/OldMotusTests.cs b/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/OldMotusTests.cs new file mode 100644 index 0000000..3d9125c --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.OldMotus.Tests/OldMotusTests.cs @@ -0,0 +1,33 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Motus.Fixtures.OldMotus.Tests; + +[TestClass] +public class OldMotusTests +{ + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + public void Passes() => Assert.AreEqual(2, 1 + 1); + + [TestMethod] + [DataRow("a")] + [DataRow("b")] + public void Rows(string value) => Assert.IsTrue(value is "a" or "b"); + + [TestMethod] + public void HasContext() => Assert.AreEqual(nameof(HasContext), TestContext.TestName); + + // Fails only for the test host that TestHostIsolationTests starts, so that the runner's verdict + // can be checked against a known failure; under dotnet test it passes. + [TestMethod] + public void FailsWhenAsked() + { + if (Environment.GetEnvironmentVariable("MOTUS_OLD_MOTUS_FIXTURE_FAIL") == "1") + Assert.Fail("failed as asked"); + } + + [TestMethod] + public void UsesItsOwnMotus() + => Assert.AreEqual(new Version(1, 0, 1, 0), typeof(Motus.Testing.MSTest.MotusTestBase).Assembly.GetName().Version); +} diff --git a/tests/Fixtures/Motus.Fixtures.WebApp.Tests/Motus.Fixtures.WebApp.Tests.csproj b/tests/Fixtures/Motus.Fixtures.WebApp.Tests/Motus.Fixtures.WebApp.Tests.csproj new file mode 100644 index 0000000..f9f51ba --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.WebApp.Tests/Motus.Fixtures.WebApp.Tests.csproj @@ -0,0 +1,33 @@ + + + + + + net8.0;net10.0 + latest + enable + enable + false + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Fixtures/Motus.Fixtures.WebApp.Tests/WebAppTests.cs b/tests/Fixtures/Motus.Fixtures.WebApp.Tests/WebAppTests.cs new file mode 100644 index 0000000..d397141 --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.WebApp.Tests/WebAppTests.cs @@ -0,0 +1,36 @@ +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Motus.Fixtures.WebApp.Tests; + +[TestClass] +public class WebAppTests +{ + private static WebApplicationFactory s_app = null!; + + // WebApplicationFactory looks for Motus.Fixtures.WebApp.deps.json under AppContext.BaseDirectory. + [AssemblyInitialize] + public static void Start(TestContext _) => s_app = new WebApplicationFactory(); + + [AssemblyCleanup] + public static void Stop() => s_app?.Dispose(); + + [TestMethod] + public async Task AppUsesItsOwnSkiaSharp() + => Assert.AreEqual("3", await s_app.CreateClient().GetStringAsync("/skia")); + + [TestMethod] + public void SuiteUsesItsOwnSkiaSharp() + => Assert.AreEqual(3, typeof(SkiaSharp.SKBitmap).Assembly.GetName().Version!.Major); + + [TestMethod] + public void BaseDirectoryIsTheOutputFolder() + => Assert.AreEqual(OutputFolder, AppContext.BaseDirectory.TrimEnd(Path.DirectorySeparatorChar)); + + [TestMethod] + public void CurrentDirectoryIsTheOutputFolder() + => Assert.AreEqual(OutputFolder, Environment.CurrentDirectory.TrimEnd(Path.DirectorySeparatorChar)); + + private static string OutputFolder + => Path.GetDirectoryName(typeof(WebAppTests).Assembly.Location)!.TrimEnd(Path.DirectorySeparatorChar); +} diff --git a/tests/Fixtures/Motus.Fixtures.WebApp/Motus.Fixtures.WebApp.csproj b/tests/Fixtures/Motus.Fixtures.WebApp/Motus.Fixtures.WebApp.csproj new file mode 100644 index 0000000..e198fa8 --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.WebApp/Motus.Fixtures.WebApp.csproj @@ -0,0 +1,18 @@ + + + + + + net8.0;net10.0 + latest + enable + enable + false + + + + + + + diff --git a/tests/Fixtures/Motus.Fixtures.WebApp/Program.cs b/tests/Fixtures/Motus.Fixtures.WebApp/Program.cs new file mode 100644 index 0000000..f92a558 --- /dev/null +++ b/tests/Fixtures/Motus.Fixtures.WebApp/Program.cs @@ -0,0 +1,9 @@ +var builder = WebApplication.CreateBuilder(args); +var app = builder.Build(); + +// The SkiaSharp the app was given, by its assembly version. +app.MapGet("/skia", () => typeof(SkiaSharp.SKBitmap).Assembly.GetName().Version!.Major.ToString()); + +app.Run(); + +public partial class Program; diff --git a/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj b/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj index 72708bf..93388f7 100644 --- a/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj +++ b/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj @@ -26,4 +26,20 @@ + + + + + + <_Parameter1>OldMotusFixtureOutput + <_Parameter2>$(MSBuildThisFileDirectory)..\Fixtures\Motus.Fixtures.OldMotus.Tests\bin\$(Configuration)\ + + + <_Parameter1>WebAppFixtureOutput + <_Parameter2>$(MSBuildThisFileDirectory)..\Fixtures\Motus.Fixtures.WebApp.Tests\bin\$(Configuration)\ + + + diff --git a/tests/Motus.Cli.Tests/Services/TestDiscoveryTests.cs b/tests/Motus.Cli.Tests/Services/TestDiscoveryTests.cs index 320c2ce..8a2e209 100644 --- a/tests/Motus.Cli.Tests/Services/TestDiscoveryTests.cs +++ b/tests/Motus.Cli.Tests/Services/TestDiscoveryTests.cs @@ -6,65 +6,70 @@ namespace Motus.Cli.Tests.Services; public class TestDiscoveryTests { [TestMethod] - public void Discover_SelfAssembly_FindsTests() + public async Task Discover_SelfAssembly_FindsTests() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], null); + var tests = await discovery.DiscoverAsync([assemblyPath], null); Assert.IsTrue(tests.Count > 0, "Should discover tests from own assembly"); } [TestMethod] - public void Discover_SelfAssembly_ContainsThisTest() + public async Task Discover_SelfAssembly_ContainsThisTest() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], null); + var tests = await discovery.DiscoverAsync([assemblyPath], null); - Assert.IsTrue(tests.Any(t => t.FullName.Contains(nameof(Discover_SelfAssembly_ContainsThisTest)))); + var test = tests.Single(t => t.FullName.EndsWith("." + nameof(Discover_SelfAssembly_ContainsThisTest), StringComparison.Ordinal)); + Assert.AreEqual(typeof(TestDiscoveryTests).FullName, test.ClassName); + Assert.AreEqual(nameof(Discover_SelfAssembly_ContainsThisTest), test.MethodName); + Assert.AreEqual(assemblyPath, test.AssemblyPath); + Assert.AreEqual(typeof(TestDiscoveryTests).Assembly.GetName().Name, test.AssemblyName); + Assert.IsNull(test.TestClass, "a discovered test lives in its test host, not in this process"); } [TestMethod] - public void Discover_FilterSubstring_FiltersCorrectly() + public async Task Discover_FilterSubstring_FiltersCorrectly() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], "ProtocolDiffer"); + var tests = await discovery.DiscoverAsync([assemblyPath], "ProtocolDiffer"); Assert.IsTrue(tests.Count > 0); Assert.IsTrue(tests.All(t => t.FullName.Contains("ProtocolDiffer"))); } [TestMethod] - public void Discover_FilterNoMatch_ReturnsEmpty() + public async Task Discover_FilterNoMatch_ReturnsEmpty() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], "ThisTestDoesNotExist_XYZ_123"); + var tests = await discovery.DiscoverAsync([assemblyPath], "ThisTestDoesNotExist_XYZ_123"); Assert.AreEqual(0, tests.Count); } [TestMethod] - public void Discover_NonexistentAssembly_PrintsError() + public async Task Discover_NonexistentAssembly_PrintsError() { - var discovery = new TestDiscovery(); - var tests = discovery.Discover(["/nonexistent/path.dll"], null); + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync(["/nonexistent/path.dll"], null); Assert.AreEqual(0, tests.Count); } [TestMethod] - public void Discover_IgnoredMethod_MarkedAsIgnored() + public async Task Discover_IgnoredMethod_MarkedAsIgnored() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], nameof(IgnoredFixture)); + var tests = await discovery.DiscoverAsync([assemblyPath], nameof(IgnoredFixture)); var ignored = tests.FirstOrDefault(t => t.FullName.Contains(nameof(IgnoredFixture.SkippedTest))); Assert.IsNotNull(ignored, "Should discover the ignored test"); @@ -76,24 +81,24 @@ public void Discover_IgnoredMethod_MarkedAsIgnored() } [TestMethod] - public void Discover_IgnoredClass_AllTestsMarkedAsIgnored() + public async Task Discover_IgnoredClass_AllTestsMarkedAsIgnored() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], nameof(IgnoredClassFixture)); + var tests = await discovery.DiscoverAsync([assemblyPath], nameof(IgnoredClassFixture)); Assert.IsTrue(tests.Count > 0, "Should discover tests from ignored class"); Assert.IsTrue(tests.All(t => t.IsIgnored), "All tests from [Ignore] class should be ignored"); } [TestMethod] - public void Discover_QuarantinedMethod_MarkedAsQuarantined() + public async Task Discover_QuarantinedMethod_MarkedAsQuarantined() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], nameof(QuarantinedFixture)); + var tests = await discovery.DiscoverAsync([assemblyPath], nameof(QuarantinedFixture)); var quarantined = tests.FirstOrDefault(t => t.FullName.Contains(nameof(QuarantinedFixture.QuarantinedTest))); Assert.IsNotNull(quarantined, "Should discover the quarantined test"); @@ -105,12 +110,12 @@ public void Discover_QuarantinedMethod_MarkedAsQuarantined() } [TestMethod] - public void Discover_QuarantinedClass_AllTestsMarkedAsQuarantined() + public async Task Discover_QuarantinedClass_AllTestsMarkedAsQuarantined() { - var discovery = new TestDiscovery(); + await using var discovery = new TestDiscovery(); var assemblyPath = typeof(TestDiscoveryTests).Assembly.Location; - var tests = discovery.Discover([assemblyPath], nameof(QuarantinedClassFixture)); + var tests = await discovery.DiscoverAsync([assemblyPath], nameof(QuarantinedClassFixture)); Assert.IsTrue(tests.Count > 0, "Should discover tests from quarantined class"); Assert.IsTrue(tests.All(t => t.Quarantined), "All tests from [Quarantine] class should be quarantined"); diff --git a/tests/Motus.Cli.Tests/Services/TestHostIsolationTests.cs b/tests/Motus.Cli.Tests/Services/TestHostIsolationTests.cs new file mode 100644 index 0000000..6947a73 --- /dev/null +++ b/tests/Motus.Cli.Tests/Services/TestHostIsolationTests.cs @@ -0,0 +1,353 @@ +using System.CommandLine; +using System.Reflection; +using Motus.Abstractions; +using Motus.Cli.Commands; +using Motus.Cli.Services; + +namespace Motus.Cli.Tests.Services; + +/// +/// motus run runs each test assembly in a test host process of its own, started from the +/// assembly's output folder with its own dependencies, so that a suite runs as it does under +/// dotnet test rather than inside the runner. +/// +[TestClass] +public class TestHostIsolationTests +{ + private const string CrashVariable = "MOTUS_TEST_HOST_CRASH_FIXTURE"; + + /// + /// The fixture suite starts an ASP.NET Core app with WebApplicationFactory, which needs the + /// suite's output folder as its base and current directory, and depends on SkiaSharp 3 where the + /// runner depends on SkiaSharp 2. Loaded into the runner, every one of its tests failed. + /// + [TestMethod] + public async Task WebAppSuite_RunsWithItsOwnDependenciesAndFolder() + { + var reporter = new RecordingReporter(); + await using var discovery = new TestDiscovery(); + + var tests = await discovery.DiscoverAsync([WebAppFixturePath(CurrentFramework)], null); + var result = await new TestRunner(2).RunAsync(tests, reporter); + + Assert.AreEqual(4, tests.Count, string.Join(" | ", tests.Select(t => t.FullName))); + Assert.AreEqual(4, result.Passed, reporter.Failures); + Assert.AreEqual("Motus.Fixtures.WebApp.Tests", reporter.SuiteName); + } + + /// + /// The runner is built for the newest .NET it supports and cannot load on an older one, so a + /// suite built for an older one runs on the runner's. + /// + [TestMethod] + public async Task SuiteBuiltForAnOlderDotNet_RunsOnTheRunnersDotNet() + { + if (Environment.Version.Major <= 8) + Assert.Inconclusive("Runs only where the runner is built for a newer .NET than the fixture's oldest."); + + var path = WebAppFixturePath("net8.0"); + if (!File.Exists(path)) + Assert.Inconclusive($"The net8.0 fixture has not been built: {path}"); + + var reporter = new RecordingReporter(); + await using var discovery = new TestDiscovery(); + + var tests = await discovery.DiscoverAsync([path], null); + var result = await new TestRunner(1).RunAsync(tests, reporter); + + Assert.AreEqual(4, result.Passed, reporter.Failures); + } + + /// + /// A test that ends the process takes its test host with it, not the runner: the test it was + /// running fails saying so, and so does each test of that assembly after it. + /// + [TestMethod] + public async Task TestHostThatExits_FailsTheRunningTestAndTheRest() + { + var reporter = new RecordingReporter(); + Environment.SetEnvironmentVariable(CrashVariable, "1"); + try + { + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([typeof(TestHostCrashFixture).Assembly.Location], nameof(TestHostCrashFixture)); + tests = tests.OrderBy(t => t.MethodName == nameof(TestHostCrashFixture.ExitsTheProcess) ? 0 : 1).ToList(); + + var result = await new TestRunner(1).RunAsync(tests, reporter); + + Assert.AreEqual(2, result.Failed, reporter.Failures); + StringAssert.Contains(reporter.ErrorOf(nameof(TestHostCrashFixture.ExitsTheProcess)), "exited unexpectedly with code 3"); + StringAssert.Contains(reporter.ErrorOf(nameof(TestHostCrashFixture.RunsAfterTheExit)), "exited unexpectedly with code 3"); + StringAssert.Contains(reporter.ErrorOf(nameof(TestHostCrashFixture.RunsAfterTheExit)), "did not run"); + } + finally + { + Environment.SetEnvironmentVariable(CrashVariable, null); + } + } + + [TestMethod] + public async Task FileThatIsNotAnAssembly_IsSkipped() + { + var path = Path.Combine(Path.GetTempPath(), $"motus-not-an-assembly-{Guid.NewGuid():N}.dll"); + await File.WriteAllTextAsync(path, "not an assembly"); + try + { + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([path], null); + + Assert.AreEqual(0, tests.Count); + CollectionAssert.AreEqual(new[] { path }, discovery.FailedAssemblies.ToArray()); + } + finally + { + File.Delete(path); + } + } + + /// + /// An assembly whose test host cannot list its tests fails the run. Before, it was reported on + /// standard error and the run exited 0 when nothing else failed, so it looked green. + /// + [TestMethod] + public async Task AssemblyThatCannotBeLoaded_FailsTheRun() + { + var path = Path.Combine(Path.GetTempPath(), $"motus-not-an-assembly-{Guid.NewGuid():N}.dll"); + await File.WriteAllTextAsync(path, "not an assembly"); + try + { + var exitCode = await RunCommand.Build().Parse([path, "--reporter", "console"]).InvokeAsync(); + + Assert.AreEqual(1, exitCode); + } + finally + { + File.Delete(path); + } + } + + /// + /// The test host runs a suite on the Motus its project references, and the oldest published + /// package lacks most of what the runner reaches into Motus for. Its tests still run and report + /// their own results; only the extras go. Before, a suite on 1.0.10 crashed the test host and + /// one on 1.0.11 to 1.0.14 failed every test with a TypeLoadException. + /// + [TestMethod] + public async Task SuiteOnTheOldestPublishedMotus_RunsWithItsOwnResults() + { + var reporter = new RecordingReporter(); + Environment.SetEnvironmentVariable("MOTUS_OLD_MOTUS_FIXTURE_FAIL", "1"); + try + { + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([FixturePath("OldMotusFixtureOutput", "Motus.Fixtures.OldMotus.Tests.dll", CurrentFramework)], null); + var result = await new TestRunner(1).RunAsync(tests, reporter); + + Assert.AreEqual(6, tests.Count, string.Join(" | ", tests.Select(t => t.FullName))); + Assert.AreEqual(5, result.Passed, reporter.Failures); + Assert.AreEqual(1, result.Failed, reporter.Failures); + StringAssert.Contains(reporter.ErrorOf("FailsWhenAsked"), "failed as asked"); + } + finally + { + Environment.SetEnvironmentVariable("MOTUS_OLD_MOTUS_FIXTURE_FAIL", null); + } + } + + /// Overloads share a name but not an identity, so each runs, rather than one of them twice. + [TestMethod] + public async Task OverloadedTests_EachRun() + { + var reporter = new RecordingReporter(); + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([typeof(OverloadFixture).Assembly.Location], nameof(OverloadFixture)); + + var result = await new TestRunner(1).RunAsync(tests, reporter); + + Assert.AreEqual(2, tests.Select(t => t.TestId).Distinct().Count(), string.Join(" | ", tests.Select(t => t.TestId))); + CollectionAssert.AreEquivalent( + new[] { $"{typeof(OverloadFixture).FullName}.Overloaded()", $"{typeof(OverloadFixture).FullName}.Overloaded(Int32)" }, + tests.Select(t => t.FullName).ToArray(), + "overloads get names of their own, which reporters and results key on"); + Assert.AreEqual(1, result.Passed, "the overload without parameters passes"); + Assert.AreEqual(1, result.Failed, "the overload with one fails"); + } + + /// + /// Each shard is a separate motus run with test hosts of its own, where a data source can name its + /// rows differently. Shards are assigned by the rows' identity, so every process agrees on them. + /// + [TestMethod] + public async Task Shards_AreTheSameInEveryProcess_WhenRowNamesChange() + { + await using var first = new TestDiscovery(); + await using var second = new TestDiscovery(); + var path = typeof(ChangingRowNamesFixture).Assembly.Location; + + var once = await first.DiscoverAsync([path], nameof(ChangingRowNamesFixture)); + var again = await second.DiscoverAsync([path], nameof(ChangingRowNamesFixture)); + + Assert.AreEqual(ChangingRowNamesFixture.RowCount, once.Count); + Assert.IsFalse(once.Select(t => t.FullName).Intersect(again.Select(t => t.FullName)).Any(), "the rows' names should differ between processes"); + CollectionAssert.AreEqual( + ShardSelector.Select(once, 1, 2).Select(t => t.TestId).ToList(), + ShardSelector.Select(again, 1, 2).Select(t => t.TestId).ToList()); + } + + /// + /// Tests run by several workers finish on different threads, and the file reporters keep plain + /// lists. Every result has to reach them: with the callbacks unsynchronized, a quick suite + /// under --workers auto lost most of its results from the JUnit and TRX files. + /// + [TestMethod] + public async Task ManyQuickTestsOnSeveralWorkers_AllReachTheFileReporters() + { + var directory = Path.Combine(Path.GetTempPath(), $"motus-reporters-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var junit = Path.Combine(directory, "results.xml"); + var trx = Path.Combine(directory, "results.trx"); + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([typeof(ManyQuickTestsFixture).Assembly.Location], nameof(ManyQuickTestsFixture)); + + var result = await new TestRunner(8).RunAsync( + tests, new Motus.Cli.Services.Reporters.CompositeReporter([new Motus.Cli.Services.Reporters.JUnitReporter(junit), new Motus.Cli.Services.Reporters.TrxReporter(trx)])); + + Assert.AreEqual(ManyQuickTestsFixture.RowCount, result.Passed); + var junitCases = System.Xml.Linq.XDocument.Load(junit).Descendants("testcase").Count(); + var trxResults = System.Xml.Linq.XDocument.Load(trx).Descendants().Count(e => e.Name.LocalName == "UnitTestResult"); + Assert.AreEqual(ManyQuickTestsFixture.RowCount, junitCases, "JUnit test cases"); + Assert.AreEqual(ManyQuickTestsFixture.RowCount, trxResults, "TRX results"); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + private static string CurrentFramework => $"net{Environment.Version.Major}.0"; + + private static string WebAppFixturePath(string framework) + => FixturePath("WebAppFixtureOutput", "Motus.Fixtures.WebApp.Tests.dll", framework); + + private static string FixturePath(string key, string fileName, string framework) + { + var output = typeof(TestHostIsolationTests).Assembly.GetCustomAttributes() + .Single(a => a.Key == key).Value!; + return Path.GetFullPath(Path.Combine(output, framework, fileName)); + } + + private sealed class RecordingReporter : IReporter + { + private readonly Dictionary _errors = new(StringComparer.Ordinal); + + public string? SuiteName { get; private set; } + + public string Failures + { + get + { + lock (_errors) + return string.Join(" | ", _errors.Where(e => e.Value is not null).Select(e => $"{e.Key}: {e.Value}")); + } + } + + public string ErrorOf(string methodName) + { + lock (_errors) + return _errors.Single(e => e.Key.EndsWith("." + methodName, StringComparison.Ordinal)).Value ?? ""; + } + + public Task OnTestRunStartAsync(TestSuiteInfo suite) + { + SuiteName = suite.SuiteName; + return Task.CompletedTask; + } + + public Task OnTestStartAsync(TestInfo test) => Task.CompletedTask; + + public Task OnTestEndAsync(TestInfo test, Motus.Abstractions.TestResult result) + { + lock (_errors) + _errors[test.TestName] = result.Passed ? null : result.ErrorMessage; + return Task.CompletedTask; + } + + public Task OnTestRunEndAsync(TestRunSummary summary) => Task.CompletedTask; + } +} + +/// +/// Ends its process when MOTUS_TEST_HOST_CRASH_FIXTURE is set, as the test above sets it for +/// the test host it starts. Under dotnet test both tests pass. +/// +[TestClass] +public class TestHostCrashFixture +{ + [TestMethod] + public void ExitsTheProcess() + { + if (Environment.GetEnvironmentVariable("MOTUS_TEST_HOST_CRASH_FIXTURE") == "1") + Environment.Exit(3); + } + + [TestMethod] + public void RunsAfterTheExit() + { + } +} + +/// +/// A data source that names its rows differently each time it is read, as one that puts a Guid or a +/// time in a row does. +/// +[TestClass] +public class ChangingRowNamesFixture +{ + public const int RowCount = 8; + + public static IEnumerable Rows => Enumerable.Range(0, RowCount).Select(i => new object[] { i, Guid.NewGuid().ToString("N") }); + + [TestMethod] + [DynamicData(nameof(Rows))] + public void RunsEachRow(int index, string token) => Assert.IsTrue(index < RowCount && token.Length == 32); +} + +/// Marks a test class for the runners, which match by name, but not for MSTest. +[AttributeUsage(AttributeTargets.Class)] +public sealed class TestFixtureAttribute : Attribute; + +/// Marks a test method of for the runners, but not for MSTest. +[AttributeUsage(AttributeTargets.Method)] +public sealed class TestAttribute : Attribute; + +/// Two test methods with one name, as NUnit allows. +[TestFixture] +public class OverloadFixture +{ + [Test] + public void Overloaded() + { + } + + [Test] + public void Overloaded(int value = 1) => throw new InvalidOperationException($"the overload with a parameter ran ({value})"); +} + +/// Many tests that finish at once, for the reporter test above. +[TestClass] +public class ManyQuickTestsFixture +{ + public const int RowCount = 60; + + public static IEnumerable Rows => Enumerable.Range(0, RowCount).Select(i => new object[] { i }); + + [TestMethod] + [DynamicData(nameof(Rows))] + public async Task FinishesQuickly(int index) + { + await Task.Yield(); + Assert.IsTrue(index < RowCount); + } +} diff --git a/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs b/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs index f7f3f92..b066de4 100644 --- a/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs +++ b/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Reflection; using Motus.Abstractions; using Motus.Cli.Commands; @@ -251,8 +250,8 @@ public async Task ThrowingDerivedCleanup_StillRunsTheBaseCleanup() [TestMethod] public async Task DataRows_AreDiscoveredAsOneTestEach_AndRunWithTheirArguments() { - DataRowFixture.Seen.Clear(); - var tests = new TestDiscovery().Discover([typeof(DataRowFixture).Assembly.Location], nameof(DataRowFixture)); + await using var discovery = new TestDiscovery(); + var tests = await discovery.DiscoverAsync([typeof(DataRowFixture).Assembly.Location], nameof(DataRowFixture)); var names = tests.Select(t => t.FullName[(typeof(DataRowFixture).FullName!.Length + 1)..]).OrderBy(n => n, StringComparer.Ordinal).ToList(); Assert.AreEqual(5, names.Count, string.Join(" | ", names)); @@ -261,18 +260,17 @@ public async Task DataRows_AreDiscoveredAsOneTestEach_AndRunWithTheirArguments() Assert.AreEqual(2, names.Count(n => n.StartsWith(nameof(DataRowFixture.Squares) + " (", StringComparison.Ordinal)), string.Join(" | ", names)); Assert.AreEqual(names.Count, names.Distinct().Count(), "each row needs a name of its own"); + // The rows run in the test host, where each test checks the arguments it was given. var result = await new TestRunner(1).RunAsync(tests, Reporter); Assert.AreEqual(5, result.Passed, $"failed: {result.Failed}"); - CollectionAssert.AreEquivalent( - new[] { "role owner", "role staff", "adds 1+2", "square 2", "square 3" }, - DataRowFixture.Seen.ToArray()); } [TestMethod] - public void QuarantineList_NamesOneRow_OrEveryRowByTheMethod() + public async Task QuarantineList_NamesOneRow_OrEveryRowByTheMethod() { - var rows = new TestDiscovery().Discover([typeof(DataRowFixture).Assembly.Location], $"{nameof(DataRowFixture)}.{nameof(DataRowFixture.SignsIn)}"); + await using var discovery = new TestDiscovery(); + var rows = await discovery.DiscoverAsync([typeof(DataRowFixture).Assembly.Location], $"{nameof(DataRowFixture)}.{nameof(DataRowFixture.SignsIn)}"); var owner = rows.Single(t => t.FullName.Contains("owner")); var staff = rows.Single(t => t.FullName.Contains("staff")); @@ -300,30 +298,20 @@ private sealed class NullReporter : IReporter [TestClass] public class DataRowFixture { - public static ConcurrentBag Seen { get; } = []; - public static IEnumerable SquareRows => [[2, 4], [3, 9]]; [TestMethod] [DataRow("owner")] [DataRow("staff")] - public void SignsIn(string role) => Seen.Add($"role {role}"); + public void SignsIn(string role) => Assert.IsTrue(role is "owner" or "staff", role); [DataTestMethod] [DataRow(1, 2, 3)] - public void Adds(int a, int b, int sum) - { - Assert.AreEqual(sum, a + b); - Seen.Add($"adds {a}+{b}"); - } + public void Adds(int a, int b, int sum) => Assert.AreEqual(sum, a + b); [TestMethod] [DynamicData(nameof(SquareRows))] - public void Squares(int n, int square) - { - Assert.AreEqual(square, n * n); - Seen.Add($"square {n}"); - } + public void Squares(int n, int square) => Assert.AreEqual(square, n * n); } /// diff --git a/tests/Motus.Tests/Motus.Tests.csproj b/tests/Motus.Tests/Motus.Tests.csproj index a3460bc..682c12a 100644 --- a/tests/Motus.Tests/Motus.Tests.csproj +++ b/tests/Motus.Tests/Motus.Tests.csproj @@ -31,6 +31,16 @@ + + + + + <_Parameter1>OldMotusFixtureOutput + <_Parameter2>$(MSBuildThisFileDirectory)..\Fixtures\Motus.Fixtures.OldMotus.Tests\bin\$(Configuration)\ + + + diff --git a/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs b/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs index 5cbad02..7d01b27 100644 --- a/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs +++ b/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Motus.Runner.Services; using Motus.Runner.Services.Models; +using Motus.Runner.Services.Timeline; namespace Motus.Tests.Runner; @@ -113,11 +114,11 @@ public async Task ClassInitialize_RunsWithATestContext() } [TestMethod] - public void Discovery_ExpandsDataRows() + public async Task Discovery_ExpandsDataRows() { var discovery = new TestDiscovery(NullLogger.Instance); - var tests = discovery.Discover([typeof(RunnerDataRowFixture).Assembly.Location], nameof(RunnerDataRowFixture)); + var tests = await discovery.DiscoverAsync([typeof(RunnerDataRowFixture).Assembly.Location], nameof(RunnerDataRowFixture)); CollectionAssert.AreEquivalent( new[] @@ -126,8 +127,153 @@ public void Discovery_ExpandsDataRows() $"{typeof(RunnerDataRowFixture).FullName}.Doubles (2,4)", }, tests.Select(t => t.FullName).ToArray()); - Assert.IsTrue(tests.All(t => t.Arguments is { Length: 2 })); + Assert.IsTrue(tests.All(t => t.ClassFullName == typeof(RunnerDataRowFixture).FullName && t.MethodName == nameof(RunnerDataRowFixture.Doubles))); + Assert.IsTrue(tests.All(t => t.CodeBase == typeof(RunnerDataRowFixture).Assembly.Location)); } + + [TestMethod] + public async Task DiscoveredDataRows_RunInATestHost_WithTheirArguments() + { + var discovery = new TestDiscovery(NullLogger.Instance); + var tests = await discovery.DiscoverAsync([typeof(RunnerDataRowFixture).Assembly.Location], nameof(RunnerDataRowFixture)); + + var states = new List(); + await CreateService().ExecuteAsync(tests, s => { lock (states) states.Add(s); }, CancellationToken.None); + + var finished = states.Where(s => s.Status != TestStatus.Running).ToList(); + Assert.AreEqual(2, finished.Count); + Assert.IsTrue(finished.All(s => s.Status == TestStatus.Passed), string.Join(" | ", finished.Select(s => s.ErrorMessage))); + } + + /// + /// A run starts a fresh test host, which reads the data source again. A row that names itself + /// differently on each read (a Guid in it) is still found, by its place in its data source. + /// + [TestMethod] + public async Task DataRowsNamedDifferentlyEachRead_RunInAFreshTestHost() + { + var discovery = new TestDiscovery(NullLogger.Instance); + var tests = await discovery.DiscoverAsync([typeof(RunnerChangingRowsFixture).Assembly.Location], nameof(RunnerChangingRowsFixture)); + + var states = new List(); + await CreateService().ExecuteAsync(tests, s => { lock (states) states.Add(s); }, CancellationToken.None); + + var finished = states.Where(s => s.Status != TestStatus.Running).ToList(); + Assert.AreEqual(3, finished.Count); + Assert.IsTrue(finished.All(s => s.Status == TestStatus.Passed), string.Join(" | ", finished.Select(s => s.ErrorMessage))); + } + + [TestMethod] + public async Task DataRowTheSourceNoLongerReturns_SaysSo() + { + var method = typeof(RunnerChangingRowsFixture).GetMethod(nameof(RunnerChangingRowsFixture.RunsEachRow))!; + var cases = Motus.Runner.Services.Execution.TestCaseDiscovery.Discover(typeof(RunnerChangingRowsFixture).Assembly, nameof(RunnerChangingRowsFixture)); + var host = new Motus.Runner.Services.Execution.InProcessTestHost(typeof(RunnerChangingRowsFixture).Assembly, cases); + var gone = Motus.Runner.Services.Execution.TestCaseExpander.RowId( + Motus.Runner.Services.Execution.TestCaseExpander.MethodId(typeof(RunnerChangingRowsFixture), method), 0, 7); + + var report = await host.RunAsync(gone, "RunnerChangingRowsFixture.RunsEachRow (7, gone)"); + + Assert.IsFalse(report.Passed); + StringAssert.Contains(report.ErrorMessage, "row 8 of its data source, which now returns 3 row(s)"); + } + + /// + /// After a rebuild that removed a [DataRow], the row's old id belongs to the row that moved up. + /// A row is found by its name when the name is there, and a [DataRow]'s name never changes, so + /// the moved row is not run in its place under the old name. + /// + [TestMethod] + public async Task DataRowAfterARebuild_IsFoundByName_NotByItsOldPlace() + { + var assembly = typeof(RunnerReorderedRowsFixture).Assembly; + var cases = Motus.Runner.Services.Execution.TestCaseDiscovery.Discover(assembly, nameof(RunnerReorderedRowsFixture)); + var host = new Motus.Runner.Services.Execution.InProcessTestHost(assembly, cases); + var firstRowId = cases.Single(c => c.Id.EndsWith("#0:0", StringComparison.Ordinal)).Id; + var className = typeof(RunnerReorderedRowsFixture).FullName; + + // As listed before the rebuild, the first row was "removed", which is gone now. + var removed = await host.RunAsync(firstRowId, $"{className}.Runs (\"removed\")"); + // And "kept" was the second row; now it is the first, and it is still found by its name. + var kept = await host.RunAsync($"{firstRowId[..^3]}1:0", cases.Single(c => c.Id == firstRowId).FullName); + + Assert.IsFalse(removed.Passed, "the row that moved up must not run under the removed row's name"); + StringAssert.Contains(removed.ErrorMessage, "data rows have changed"); + Assert.IsTrue(kept.Passed, kept.ErrorMessage); + Assert.AreEqual(cases.Single(c => c.Id == firstRowId).FullName, kept.FullName); + } + + /// + /// An assembly whose test host cannot start fails its own tests, and the other assemblies in the + /// run still run, rather than the whole run stopping with the rest left as they were. + /// + [TestMethod] + public async Task AssemblyThatCannotStart_FailsItsTests_AndTheOthersRun() + { + var path = Path.Combine(Path.GetTempPath(), $"motus-not-an-assembly-{Guid.NewGuid():N}.dll"); + await File.WriteAllTextAsync(path, "not an assembly"); + try + { + var broken = new DiscoveredTest(null, null, "Broken.Tests.Test", "Broken", false, CodeBase: path); + var discovery = new TestDiscovery(NullLogger.Instance); + var good = await discovery.DiscoverAsync([typeof(RunnerDataRowFixture).Assembly.Location], nameof(RunnerDataRowFixture)); + + var states = new List(); + await CreateService().ExecuteAsync([broken, .. good], s => { lock (states) states.Add(s); }, CancellationToken.None); + + var finished = states.Where(s => s.Status != TestStatus.Running).ToDictionary(s => s.FullName); + Assert.AreEqual(TestStatus.Failed, finished[broken.FullName].Status); + StringAssert.Contains(finished[broken.FullName].ErrorMessage, "Could not run the tests in Broken"); + Assert.IsTrue(good.All(t => finished[t.FullName].Status == TestStatus.Passed)); + } + finally + { + File.Delete(path); + } + } + + /// + /// The visual runner's test host also installs its browser hooks, which reach further into Motus + /// than a plain run. On the oldest published Motus the tests still run with their own results. + /// + [TestMethod] + public async Task SuiteOnTheOldestPublishedMotus_RunsInTheVisualRunner() + { + var output = typeof(TestExecutionServiceTests).Assembly.GetCustomAttributes() + .Single(a => a.Key == "OldMotusFixtureOutput").Value!; + var path = Path.GetFullPath(Path.Combine(output, $"net{Environment.Version.Major}.0", "Motus.Fixtures.OldMotus.Tests.dll")); + var timeline = new TimelineService(); + var service = new TestExecutionService(NullLogger.Instance, null, new ScreencastService(), timeline, new StepDebugService()); + + var tests = await new TestDiscovery(NullLogger.Instance).DiscoverAsync([path], null); + var states = new List(); + await service.ExecuteAsync(tests, s => { lock (states) states.Add(s); }, CancellationToken.None); + + var finished = states.Where(s => s.Status != TestStatus.Running).ToList(); + Assert.AreEqual(6, finished.Count); + Assert.AreEqual(6, finished.Count(s => s.Status == TestStatus.Passed), string.Join(" | ", finished.Select(s => $"{s.FullName}: {s.ErrorMessage}"))); + } +} + +/// [DataRow]s, standing for what is left after a rebuild removed one, for the test above. +[TestClass] +public class RunnerReorderedRowsFixture +{ + [TestMethod] + [DataRow("kept")] + [DataRow("other")] + public void Runs(string value) => Assert.IsTrue(value is "kept" or "other"); +} + +/// Rows whose names change on every read of the data source, run by the visual runner above. +[TestClass] +public class RunnerChangingRowsFixture +{ + public static IEnumerable Rows => Enumerable.Range(0, 3).Select(i => new object[] { i, Guid.NewGuid().ToString("N") }); + + [TestMethod] + [DynamicData(nameof(Rows))] + public void RunsEachRow(int index, string token) => Assert.IsTrue(index < 3 && token.Length == 32); } /// A data-driven test MSTest runs as well, discovered by the visual runner in the test above. diff --git a/tests/Motus.Tests/Runner/TestHostLauncherTests.cs b/tests/Motus.Tests/Runner/TestHostLauncherTests.cs new file mode 100644 index 0000000..fdf350b --- /dev/null +++ b/tests/Motus.Tests/Runner/TestHostLauncherTests.cs @@ -0,0 +1,49 @@ +using Motus.Runner.Services.Execution; + +namespace Motus.Tests.Runner; + +[TestClass] +public class TestHostLauncherTests +{ + [TestMethod] + [DataRow("net8.0", true)] + [DataRow("net8.0-windows", true)] + [DataRow("net10.0", false)] + [DataRow("net11.0", false)] + [DataRow("netcoreapp3.1", true)] + public void NeedsNewerRuntime_WhenTheSuiteTargetsAnOlderDotNetThanTheRunner(string tfm, bool expectedOnNet10) + { + var path = Path.Combine(Path.GetTempPath(), $"motus-{Guid.NewGuid():N}.runtimeconfig.json"); + File.WriteAllText(path, $$"""{ "runtimeOptions": { "tfm": "{{tfm}}" } }"""); + try + { + // The runner here is the build for the framework these tests run on. + var expected = Environment.Version.Major >= 10 ? expectedOnNet10 : tfm == "netcoreapp3.1"; + Assert.AreEqual(expected, TestHostLauncher.NeedsNewerRuntime(path)); + } + finally + { + File.Delete(path); + } + } + + [TestMethod] + public void NeedsNewerRuntime_WhenTheRuntimeConfigCannotBeRead() + => Assert.IsTrue(TestHostLauncher.NeedsNewerRuntime(Path.Combine(Path.GetTempPath(), $"missing-{Guid.NewGuid():N}.json"))); + + [TestMethod] + public void StartInfo_UsesTheSuitesOwnDepsAndRuntimeConfig() + { + var assembly = typeof(TestHostLauncherTests).Assembly.Location; + var stem = Path.Combine(Path.GetDirectoryName(assembly)!, Path.GetFileNameWithoutExtension(assembly)); + + var startInfo = TestHostLauncher.CreateStartInfo("pipe", assembly, visual: true); + var args = startInfo.ArgumentList.ToList(); + + Assert.AreEqual("exec", args[0]); + Assert.AreEqual(stem + ".runtimeconfig.json", args[args.IndexOf("--runtimeconfig") + 1]); + Assert.AreEqual(stem + ".deps.json", args[args.IndexOf("--depsfile") + 1]); + CollectionAssert.IsSubsetOf(new[] { TestHostServer.Command, "pipe", assembly, TestHostServer.VisualOption }, args); + Assert.IsTrue(File.Exists(startInfo.FileName), startInfo.FileName); + } +} diff --git a/tests/Motus.Tests/Runner/VisualTestHostIntegrationTests.cs b/tests/Motus.Tests/Runner/VisualTestHostIntegrationTests.cs new file mode 100644 index 0000000..4c228a0 --- /dev/null +++ b/tests/Motus.Tests/Runner/VisualTestHostIntegrationTests.cs @@ -0,0 +1,106 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Motus.Abstractions; +using Motus.Runner.Services; +using Motus.Runner.Services.Models; +using Motus.Runner.Services.Timeline; + +namespace Motus.Tests.Runner; + +/// +/// The visual runner runs tests in a test host process of their own assembly, so the browser view, +/// the timeline and step debugging all have to reach it from there. +/// +[TestClass] +[TestCategory("Integration")] +public class VisualTestHostIntegrationTests +{ + [TestMethod] + public async Task BrowserTest_InATestHost_StreamsFramesTimelineAndSteps() + { + var timeline = new TimelineService(); + var screencast = new ScreencastService(); + var stepDebug = new CountingStepDebug(); + var discovery = new TestDiscovery(NullLogger.Instance); + var executor = new TestExecutionService(NullLogger.Instance, null, screencast, timeline, stepDebug); + var session = new TestSessionService(discovery, executor, timeline); + + var frames = 0; + screencast.FrameReceived += _ => Interlocked.Increment(ref frames); + + await session.LoadAssembliesAsync([typeof(VisualTestHostFixture).Assembly.Location], nameof(VisualTestHostFixture)); + var test = session.DiscoveredTests.Single(); + Assert.IsNull(test.TestClass, "a discovered test should run in a test host"); + + await session.RunAllAsync(); + + var state = session.States[test.FullName]; + if (state.Status == TestStatus.Failed && state.ErrorMessage?.Contains("No browser found", StringComparison.Ordinal) == true) + Assert.Inconclusive("No browser found; skipping integration tests."); + + Assert.AreEqual(TestStatus.Passed, state.Status, state.ErrorMessage); + Assert.IsTrue(frames > 0, "the browser view should receive the test host's screencast"); + Assert.IsTrue(timeline.Entries.Any(e => e.ActionType == "click" && e.TestName == test.FullName), + "the timeline should record the click under the test that made it: " + + string.Join(", ", timeline.Entries.Select(e => $"{e.ActionType} ({e.TestName})"))); + CollectionAssert.AreEqual( + Enumerable.Range(0, timeline.Entries.Count).ToList(), + timeline.Entries.Select(e => e.Index).ToList(), + "entries from the test host are numbered in the order they arrive"); + Assert.IsTrue(stepDebug.Actions.Contains("click"), "each action should ask the step debugger before it runs"); + Assert.IsFalse(screencast.IsStreaming, "the browser view should stop once the test host has gone"); + } + + private sealed class CountingStepDebug : IStepDebugService + { + public List Actions { get; } = []; + public bool IsStepMode => false; + public bool IsPaused => false; + public string? PendingActionType => null; + public string? PendingSelector => null; + public event Action? StateChanged { add { } remove { } } + public void EnableStepMode() { } + public void DisableStepMode() { } + public void Advance() { } + public void Resume() { } + + public Task WaitIfPausedAsync(string actionType, string? selector, CancellationToken ct) + { + lock (Actions) + Actions.Add(actionType); + return Task.CompletedTask; + } + } +} + +/// A browser test that the visual runner runs in a test host in the test above. +[TestClass] +[TestCategory("Integration")] +public class VisualTestHostFixture +{ + [TestMethod] + public async Task ClicksAButton() + { + IBrowser browser; + try + { + browser = await MotusLauncher.LaunchAsync(new LaunchOptions { Headless = true }); + } + catch (FileNotFoundException) + { + Assert.Inconclusive("No browser found; skipping integration tests."); + return; + } + + await using (browser) + { + var page = await browser.NewPageAsync(); + await page.SetContentAsync( + "" + + "
"); + await page.Locator("button").ClickAsync(); + + // Long enough for the screencast to send frames of the animation. + await Task.Delay(1000); + } + } +}