diff --git a/docs/guides/flaky-tests-and-quarantine.md b/docs/guides/flaky-tests-and-quarantine.md index 1b55216..c95eae2 100644 --- a/docs/guides/flaky-tests-and-quarantine.md +++ b/docs/guides/flaky-tests-and-quarantine.md @@ -60,7 +60,7 @@ Or supply a list file, which quarantines without touching the source. That suits motus run bin/Release/net8.0/MyTests.dll --quarantine quarantine.txt ``` -One fully qualified test name per line. Blank lines are ignored, and a line whose first non-blank character is `#` is a comment. The whole line is the name, so a trailing `# reason` on the same line becomes part of it and the entry silently matches nothing; put the reason on its own line above. +One fully qualified test name per line. A data-driven test is listed by the name of one row, such as `MyTests.CheckoutTests.PaysBy (card)`, or by its method, `MyTests.CheckoutTests.PaysBy`, which covers every row. Blank lines are ignored, and a line whose first non-blank character is `#` is a comment. The whole line is the name, so a trailing `# reason` on the same line becomes part of it and the entry silently matches nothing; put the reason on its own line above. ``` # Flaky since the payment provider sandbox migration (#412) diff --git a/docs/guides/testing-frameworks.md b/docs/guides/testing-frameworks.md index a7c23f4..c69f0f5 100644 --- a/docs/guides/testing-frameworks.md +++ b/docs/guides/testing-frameworks.md @@ -27,7 +27,7 @@ Inherit from `MotusTestBase` to receive a fully managed `IBrowserContext` and `I |---|---| | `protected IBrowserContext Context` | The context for the current test. Available after `[TestInitialize]`. | | `protected IPage Page` | The page for the current test. Available after `[TestInitialize]`. | -| `public TestContext TestContext` | Set by the MSTest runner. Used internally to detect test outcome for failure tracing. | +| `public TestContext TestContext` | Set by MSTest, and by `motus run` and the visual runner. Used internally to detect test outcome for failure tracing. | ### Browser lifecycle diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d226c95..ef90e01 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -63,6 +63,13 @@ motus run bin/Release/net8.0/MyTests.dll --shard 1/4 --retries 2 The output path for a reporter is part of its value, so it is `--reporter junit:results.xml` rather than a separate flag. Passing `--reporter junit` with no path writes nowhere. +**How tests run.** `motus run` and the visual runner load the test assembly and call its tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on: + +- MSTest tests get a `TestContext`, through their `TestContext` property or a constructor that takes one, and `[AssemblyInitialize]`, `[ClassInitialize]` and the matching cleanup methods receive one when they take it. It carries `TestName`, `FullyQualifiedTestClassName` and a `CancellationTokenSource`, and `CurrentTestOutcome` is the test's own outcome by the time `[TestCleanup]` runs. `WriteLine` goes to the console. Files passed to `AddResultFile` are left where they are rather than attached to a result. +- `[DataRow]`, `[DynamicData]` and any other MSTest data source run as one test per row, named the way MSTest names them, such as `SignInTests.SignsIn (owner)`. `--filter` matches that name, so it can select a single row. +- Setup methods (`[TestInitialize]`, NUnit `[SetUp]`) run base class first. Cleanup methods (`[TestCleanup]`, NUnit `[TearDown]`) run derived class first and base class last, and each one runs even when an earlier one throws. +- Attributes derived from `[TestClass]` or `[TestMethod]`, such as `[DataTestMethod]`, `[MotusTestClass]` and `[MotusTestMethod]`, mark tests as the attributes they derive from do. NUnit `[TestCase]` and xUnit `[Theory]` tests are not run. + **Exit code.** `0` when everything passed. `1` when any test failed, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky. See [Sharding](../guides/sharding.md), [Flaky Tests and Quarantine](../guides/flaky-tests-and-quarantine.md), and [Configuration](../guides/configuration.md) for the settings behind these. diff --git a/src/Motus.Cli/Commands/RunCommand.cs b/src/Motus.Cli/Commands/RunCommand.cs index 3d4ca17..3c82f9e 100644 --- a/src/Motus.Cli/Commands/RunCommand.cs +++ b/src/Motus.Cli/Commands/RunCommand.cs @@ -208,7 +208,7 @@ public static Command Build() } tests = tests - .Select(t => t with { Quarantined = t.Quarantined || quarantineList.Contains(t.FullName) }) + .Select(t => t with { Quarantined = t.Quarantined || IsListed(t, quarantineList) }) .ToList(); } @@ -250,6 +250,14 @@ private static bool TryParseRetryPolicy(string spec, out RetryPolicy policy) } } + /// + /// True when the quarantine list names . A data-driven test's rows can be + /// listed one by one, or all at once by the method's name. + /// + internal static bool IsListed(DiscoveredTest test, IReadOnlySet quarantineList) + => quarantineList.Contains(test.FullName) + || quarantineList.Contains($"{test.TestClass.FullName}.{test.TestMethod.Name}"); + private static HashSet LoadQuarantineList(string path) { var set = new HashSet(StringComparer.Ordinal); diff --git a/src/Motus.Cli/Services/TestDiscovery.cs b/src/Motus.Cli/Services/TestDiscovery.cs index 0531b28..bf5dc64 100644 --- a/src/Motus.Cli/Services/TestDiscovery.cs +++ b/src/Motus.Cli/Services/TestDiscovery.cs @@ -1,8 +1,13 @@ using System.Reflection; +using Motus.Runner.Services.Execution; namespace Motus.Cli.Services; -public sealed record DiscoveredTest(Type TestClass, MethodInfo TestMethod, string FullName, bool IsIgnored, bool Quarantined = false); +/// +/// A test case found in a test assembly. is the data row of one case of a +/// data-driven test, or null for a test without one. +/// +public sealed record DiscoveredTest(Type TestClass, MethodInfo TestMethod, string FullName, bool IsIgnored, bool Quarantined = false, object?[]? Arguments = null); public sealed class TestDiscovery { @@ -19,6 +24,12 @@ public sealed class TestDiscovery "FactAttribute", }; + // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply. + private static readonly HashSet ExcludedMethodAttributes = new(StringComparer.Ordinal) + { + "TheoryAttribute", + }; + private static readonly HashSet IgnoreAttributes = new(StringComparer.Ordinal) { "IgnoreAttribute", // MSTest + NUnit @@ -63,14 +74,17 @@ public List Discover(string[] assemblyPaths, string? filter) if (!IsTestMethod(method)) continue; - var fullName = $"{type.FullName}.{method.Name}"; + var methodIgnored = classIgnored || IsIgnoredMethod(method); + var quarantined = classQuarantined || IsQuarantinedMethod(method); - if (filter is not null && !fullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) - continue; + foreach (var testCase in TestCaseExpander.Expand(type, method, message => Console.Error.WriteLine(message))) + { + if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) + continue; - var isIgnored = classIgnored || IsIgnoredMethod(method); - var quarantined = classQuarantined || IsQuarantinedMethod(method); - tests.Add(new DiscoveredTest(type, method, fullName, isIgnored, quarantined)); + var isIgnored = methodIgnored || testCase.IgnoreReason is not null; + tests.Add(new DiscoveredTest(type, method, testCase.FullName, isIgnored, quarantined, testCase.Arguments)); + } } } } @@ -84,13 +98,14 @@ private static bool IsTestClass(Type type) return false; return type.GetCustomAttributes(true) - .Any(a => ClassAttributes.Contains(a.GetType().Name)); + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes)); } private static bool IsTestMethod(MethodInfo method) { return method.GetCustomAttributes(true) - .Any(a => MethodAttributes.Contains(a.GetType().Name)); + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes) + && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes)); } private static bool IsIgnoredMethod(MethodInfo method) diff --git a/src/Motus.Cli/Services/TestRunner.cs b/src/Motus.Cli/Services/TestRunner.cs index 51ea6e1..ea71cb2 100644 --- a/src/Motus.Cli/Services/TestRunner.cs +++ b/src/Motus.Cli/Services/TestRunner.cs @@ -4,6 +4,7 @@ using Motus; using Motus.Abstractions; using Motus.Cli.Services.Reporters; +using Motus.Runner.Services.Execution; using CliTestResult = Motus.Cli.Services.TestResult; namespace Motus.Cli.Services; @@ -399,93 +400,32 @@ private static bool IsTransientCdpFailure(Exception? ex) private async Task<(CliTestResult Result, Exception? Failure)> ExecuteTestAsync(DiscoveredTest test) { - var testSw = Stopwatch.StartNew(); - object? instance = null; - try - { - instance = Activator.CreateInstance(test.TestClass)!; - - // Publish the test method name so MotusTestBase can resolve per-method attributes - // (e.g. [PerformanceBudget]) and name failure artifacts without a TestContext - TestMethodNameContext.Set(test.TestMethod.Name); - - // Run [ClassInitialize] once per type (static, takes TestContext) - if (_initializedClasses.TryAdd(test.TestClass, true)) + var run = await TestCaseInvoker.RunAsync( + test.TestClass, + test.TestMethod, + test.Arguments, + DisplayNameOf(test), + beforeInstance: async () => { - await RunStaticLifecycleMethodAsync(test.TestClass, "ClassInitializeAttribute"); - await RunStaticLifecycleMethodAsync(test.TestClass, "OneTimeSetUpAttribute"); - } - - // Run [TestInitialize] / [SetUp] - await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TestInitializeAttribute"); - await ExecuteLifecycleMethodAsync(instance, test.TestClass, "SetUpAttribute"); - - var result = test.TestMethod.Invoke(instance, null); - - if (result is Task task) - await task; - - testSw.Stop(); - return (new CliTestResult(test.FullName, true, testSw.Elapsed, null, null), null); - } - catch (TargetInvocationException ex) when (ex.InnerException is not null) - { - testSw.Stop(); - return ( - new CliTestResult(test.FullName, false, testSw.Elapsed, ex.InnerException.Message, ex.InnerException.StackTrace), - ex.InnerException); - } - catch (Exception ex) - { - testSw.Stop(); - return ( - new CliTestResult(test.FullName, false, testSw.Elapsed, ex.Message, ex.StackTrace), - ex); - } - finally - { - if (instance is not null) - { - try - { - // Run [TestCleanup] / [TearDown] - await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TestCleanupAttribute"); - await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TearDownAttribute"); - } - catch (Exception ex) + // Run [ClassInitialize] once per type (static, takes TestContext) + if (_initializedClasses.TryAdd(test.TestClass, true)) { - Console.Error.WriteLine($"TestCleanup failed for {test.FullName}: {ex.InnerException?.Message ?? ex.Message}"); + await RunStaticLifecycleMethodAsync(test.TestClass, "ClassInitializeAttribute"); + await RunStaticLifecycleMethodAsync(test.TestClass, "OneTimeSetUpAttribute"); } + }, + onCleanupFailure: ex => Console.Error.WriteLine($"TestCleanup failed for {test.FullName}: {ex.Message}")); - if (instance is IAsyncDisposable asyncDisposable) - await asyncDisposable.DisposeAsync(); - else if (instance is IDisposable disposable) - disposable.Dispose(); - } - - TestMethodNameContext.Clear(); - } + return run.Failure is null + ? (new CliTestResult(test.FullName, true, run.Duration, null, null), null) + : (new CliTestResult(test.FullName, false, run.Duration, run.Failure.Message, run.Failure.StackTrace), run.Failure); } - private static async Task ExecuteLifecycleMethodAsync(object instance, Type testClass, string attributeName) + /// The test case's name without its class, such as SignsIn ("owner"). + private static string DisplayNameOf(DiscoveredTest test) { - // Walk the hierarchy bottom-up, then reverse so base-class methods run first - var methods = new List(); - var type = testClass; - while (type is not null && type != typeof(object)) - { - var declared = type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); - methods.InsertRange(0, declared); - type = type.BaseType; - } - - foreach (var method in methods) - { - var result = method.Invoke(instance, null); - if (result is Task task) - await task; - } + var prefix = test.TestClass.FullName + "."; + return test.FullName.StartsWith(prefix, StringComparison.Ordinal) ? test.FullName[prefix.Length..] : test.TestMethod.Name; } private static async Task RunAssemblyCleanupAsync(Assembly assembly) @@ -496,11 +436,7 @@ private static async Task RunAssemblyCleanupAsync(Assembly assembly) .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyCleanupAttribute")); foreach (var method in methods) - { - var result = method.Invoke(null, null); - if (result is Task task) - await task; - } + await TestCaseInvoker.InvokeStaticAsync(method, type); } } @@ -512,13 +448,7 @@ private static async Task RunAssemblyInitializeAsync(Assembly assembly) .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyInitializeAttribute")); foreach (var method in methods) - { - var parameters = method.GetParameters(); - object?[] args = parameters.Length > 0 ? [null] : []; - var result = method.Invoke(null, args); - if (result is Task task) - await task; - } + await TestCaseInvoker.InvokeStaticAsync(method, type); } } @@ -528,12 +458,6 @@ private static async Task RunStaticLifecycleMethodAsync(Type type, string attrib .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); foreach (var method in methods) - { - var parameters = method.GetParameters(); - object?[] args = parameters.Length > 0 ? [null] : []; - var result = method.Invoke(null, args); - if (result is Task task) - await task; - } + await TestCaseInvoker.InvokeStaticAsync(method, type); } } diff --git a/src/Motus.Runner/Components/Explorer/TestMethodNode.razor b/src/Motus.Runner/Components/Explorer/TestMethodNode.razor index c10f899..aa1c619 100644 --- a/src/Motus.Runner/Components/Explorer/TestMethodNode.razor +++ b/src/Motus.Runner/Components/Explorer/TestMethodNode.razor @@ -30,7 +30,11 @@ private TestStatus CurrentStatus => CurrentState?.Status ?? TestStatus.Pending; - private string MethodDisplayName => Test.TestMethod?.Name ?? GetMethodFromFullName(Test.FullName); + // A data row's name carries its arguments, e.g. SignsIn ("owner"), so rows of one method differ. + private string MethodDisplayName => + Test.TestClass?.FullName is { } className && Test.FullName.StartsWith(className + ".", StringComparison.Ordinal) + ? Test.FullName[(className.Length + 1)..] + : Test.TestMethod?.Name ?? GetMethodFromFullName(Test.FullName); private bool ShowRerunButton => Options.ViewerMode == ViewerMode.Trx && CurrentStatus != TestStatus.Passed; diff --git a/src/Motus.Runner/Motus.Runner.csproj b/src/Motus.Runner/Motus.Runner.csproj index 5069028..d098049 100644 --- a/src/Motus.Runner/Motus.Runner.csproj +++ b/src/Motus.Runner/Motus.Runner.csproj @@ -13,6 +13,11 @@ + + + + + diff --git a/src/Motus.Runner/Services/Execution/MSTestContextFactory.cs b/src/Motus.Runner/Services/Execution/MSTestContextFactory.cs new file mode 100644 index 0000000..97dc93d --- /dev/null +++ b/src/Motus.Runner/Services/Execution/MSTestContextFactory.cs @@ -0,0 +1,361 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Reflection; +using System.Reflection.Emit; + +namespace Motus.Runner.Services.Execution; + +/// +/// Builds MSTest TestContext objects for tests run by reflection. +/// +/// +/// The runners do not reference MSTest: a test assembly brings its own version, and the +/// TestContext it expects is that version's type, which has moved between assemblies +/// and gained abstract members from one major version to the next. So the subclass is generated +/// at run time, once per TestContext type, by overriding whatever members that version +/// declares abstract, plus CurrentTestOutcome. Every override forwards to +/// . When a version cannot be subclassed this way, no context is made +/// and the test runs without one, as it did before the runners made contexts at all. +/// +internal static class MSTestContextFactory +{ + internal const string TestContextTypeName = "Microsoft.VisualStudio.TestTools.UnitTesting.TestContext"; + + private static readonly ConcurrentDictionary s_shapes = new(); + + /// True when is some version of MSTest's TestContext. + public static bool IsTestContextType(Type? type) + => type is { IsClass: true, IsAbstract: true, FullName: TestContextTypeName }; + + /// + /// Creates a describing , or returns + /// null when that is not an MSTest TestContext or this version of it can't be built. + /// + public static object? TryCreate(Type contextType, TestCaseContext testCase) + { + if (!IsTestContextType(contextType)) + return null; + + var shape = s_shapes.GetOrAdd(contextType, TryBuildShape); + if (shape is null) + return null; + + try + { + return shape.Create(testCase); + } + catch + { + return null; + } + } + + /// + /// Points MSTest's static TestContext.Current (MSTest 4.1 and later) at + /// . It is an async local, so this must be called from the async + /// method that goes on to run the test. Does nothing for versions without it. + /// + public static void SetCurrent(Type contextType, object? context) + { + if (s_shapes.TryGetValue(contextType, out var shape) && shape?.CurrentSetter is { } setter) + { + try { setter.Invoke(null, [context]); } + catch { /* the test can still use the instance it was given */ } + } + } + + /// + /// Records the exception that failed the test on TestContext.TestException + /// (MSTest 4 and later), so the test's cleanup methods can read it. + /// + public static void SetException(object context, Exception exception) + { + if (context.GetType().BaseType is { } contextType + && s_shapes.TryGetValue(contextType, out var shape) + && shape?.ExceptionSetter is { } setter) + { + try { setter.Invoke(context, [exception]); } + catch { /* informational only */ } + } + } + + private static object? Dispatch(ContextShape shape, TestCaseContext testCase, Dictionary properties, string member, object?[] args) + { + switch (member) + { + case "get_Properties": + return properties; + + case "get_CurrentTestOutcome": + return shape.ToOutcome(testCase.Outcome); + + case "Write": + Console.Out.Write(Format(args)); + return null; + + case "WriteLine": + Console.Out.WriteLine(Format(args)); + return null; + + case "DisplayMessage": + // (MessageLevel level, string message) + Console.Out.WriteLine(args.Length > 1 ? args[1] as string : null); + return null; + + default: + // AddResultFile, and any member a later MSTest makes abstract: the runners have no + // result attachments, so there is nothing to do and the default is returned. + return null; + } + } + + private static string? Format(object?[] args) + { + if (args.Length == 0) + return null; + + var format = args[0] as string; + if (args.Length > 1 && args[1] is object?[] formatArgs && format is not null) + return string.Format(CultureInfo.CurrentCulture, format, formatArgs); + + return format; + } + + private static ContextShape? TryBuildShape(Type contextType) + { + try + { + var baseConstructor = contextType.GetConstructor( + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, Type.EmptyTypes); + if (baseConstructor is null || !(baseConstructor.IsPublic || baseConstructor.IsFamily || baseConstructor.IsFamilyOrAssembly)) + return null; + + var toOverride = new List(); + foreach (var method in contextType.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) + { + if (!method.IsAbstract) + continue; + + // A member this can't override would leave the generated type abstract. + if (!CanOverride(method)) + return null; + + toOverride.Add(method); + } + + var outcomeGetter = contextType.GetProperty("CurrentTestOutcome", BindingFlags.Instance | BindingFlags.Public)?.GetGetMethod(); + if (outcomeGetter is null || !outcomeGetter.ReturnType.IsEnum) + return null; + if (!outcomeGetter.IsAbstract) + { + if (!outcomeGetter.IsVirtual || outcomeGetter.IsFinal) + return null; + toOverride.Add(outcomeGetter); + } + + var generated = Generate(contextType, baseConstructor, toOverride); + var constructor = generated.GetConstructor([typeof(Func)])!; + + return new ContextShape( + constructor, + outcomeGetter.ReturnType, + ReadPropertyLabels(contextType), + NonPublicSetter(contextType, "CancellationTokenSource", BindingFlags.Instance), + NonPublicSetter(contextType, "TestDisplayName", BindingFlags.Instance), + NonPublicSetter(contextType, "TestData", BindingFlags.Instance), + NonPublicSetter(contextType, "TestException", BindingFlags.Instance), + NonPublicSetter(contextType, "Current", BindingFlags.Static)); + } + catch + { + return null; + } + } + + private static bool CanOverride(MethodInfo method) + => !method.IsGenericMethodDefinition + && (method.IsPublic || method.IsFamily || method.IsFamilyOrAssembly) + && !method.ReturnType.IsByRef + && !method.ReturnType.IsPointer + && method.GetParameters().All(p => !p.ParameterType.IsByRef && !p.ParameterType.IsPointer); + + private static MethodInfo? NonPublicSetter(Type contextType, string propertyName, BindingFlags kind) + => contextType.GetProperty(propertyName, BindingFlags.Public | BindingFlags.NonPublic | kind)?.GetSetMethod(nonPublic: true); + + /// + /// Reads the keys MSTest uses for Properties from its own ...Label fields, so the + /// base class's TestName and friends find what is stored, whatever this version calls them. + /// + private static Dictionary ReadPropertyLabels(Type contextType) + { + var labels = new Dictionary(StringComparer.Ordinal); + foreach (var field in contextType.GetFields(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic)) + { + if (field.FieldType != typeof(string) || !field.Name.EndsWith("Label", StringComparison.Ordinal)) + continue; + + var name = field.Name[..^"Label".Length]; + labels[name] = (field.IsLiteral ? field.GetRawConstantValue() : field.GetValue(null)) as string ?? name; + } + + return labels; + } + + private static Type Generate(Type contextType, ConstructorInfo baseConstructor, List toOverride) + { + // Collectible, so that it can refer to a test assembly loaded into a collectible context. + var assemblyName = new AssemblyName($"Motus.Runner.TestContext.{Guid.NewGuid():N}"); + var assembly = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.RunAndCollect); + var module = assembly.DefineDynamicModule(assemblyName.Name!); + var type = module.DefineType( + "Motus.Runner.MotusTestContext", + TypeAttributes.Public | TypeAttributes.Sealed | TypeAttributes.Class, + contextType); + + var dispatchType = typeof(Func); + var dispatchField = type.DefineField("_dispatch", dispatchType, FieldAttributes.Private | FieldAttributes.InitOnly); + var invoke = dispatchType.GetMethod(nameof(Func.Invoke))!; + + var constructor = type.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard, [dispatchType]); + var ctorIl = constructor.GetILGenerator(); + ctorIl.Emit(OpCodes.Ldarg_0); + ctorIl.Emit(OpCodes.Call, baseConstructor); + ctorIl.Emit(OpCodes.Ldarg_0); + ctorIl.Emit(OpCodes.Ldarg_1); + ctorIl.Emit(OpCodes.Stfld, dispatchField); + ctorIl.Emit(OpCodes.Ret); + + foreach (var baseMethod in toOverride) + { + var parameters = baseMethod.GetParameters(); + var attributes = MethodAttributes.Virtual | MethodAttributes.HideBySig + | (baseMethod.IsPublic ? MethodAttributes.Public : MethodAttributes.Family); + if (baseMethod.IsSpecialName) + attributes |= MethodAttributes.SpecialName; + + var method = type.DefineMethod( + baseMethod.Name, attributes, baseMethod.ReturnType, parameters.Select(p => p.ParameterType).ToArray()); + var il = method.GetILGenerator(); + + // _dispatch(name, new object[] { arg0, arg1, ... }) + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, dispatchField); + il.Emit(OpCodes.Ldstr, baseMethod.Name); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(object)); + for (var i = 0; i < parameters.Length; i++) + { + il.Emit(OpCodes.Dup); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameters[i].ParameterType.IsValueType || parameters[i].ParameterType.IsGenericParameter) + il.Emit(OpCodes.Box, parameters[i].ParameterType); + il.Emit(OpCodes.Stelem_Ref); + } + + il.Emit(OpCodes.Callvirt, invoke); + + if (baseMethod.ReturnType == typeof(void)) + { + il.Emit(OpCodes.Pop); + il.Emit(OpCodes.Ret); + } + else + { + // A null result becomes the return type's default rather than a NullReferenceException. + var result = il.DeclareLocal(typeof(object)); + var fallback = il.DeclareLocal(baseMethod.ReturnType); + var useDefault = il.DefineLabel(); + il.Emit(OpCodes.Stloc, result); + il.Emit(OpCodes.Ldloc, result); + il.Emit(OpCodes.Brfalse_S, useDefault); + il.Emit(OpCodes.Ldloc, result); + il.Emit(OpCodes.Unbox_Any, baseMethod.ReturnType); + il.Emit(OpCodes.Ret); + il.MarkLabel(useDefault); + il.Emit(OpCodes.Ldloca, fallback); + il.Emit(OpCodes.Initobj, baseMethod.ReturnType); + il.Emit(OpCodes.Ldloc, fallback); + il.Emit(OpCodes.Ret); + } + + type.DefineMethodOverride(method, baseMethod); + } + + return type.CreateType()!; + } + + private sealed class ContextShape( + ConstructorInfo constructor, + Type outcomeType, + Dictionary labels, + MethodInfo? cancellationSetter, + MethodInfo? displayNameSetter, + MethodInfo? dataSetter, + MethodInfo? exceptionSetter, + MethodInfo? currentSetter) + { + public MethodInfo? ExceptionSetter => exceptionSetter; + + public MethodInfo? CurrentSetter => currentSetter; + + public object ToOutcome(TestCaseOutcome outcome) + => Enum.TryParse(outcomeType, outcome.ToString(), out var value) + ? value! + : Enum.ToObject(outcomeType, 0); + + public object Create(TestCaseContext testCase) + { + var properties = BuildProperties(testCase); + Func dispatch = (member, args) => Dispatch(this, testCase, properties, member, args); + var instance = constructor.Invoke([dispatch]); + + TrySet(cancellationSetter, instance, testCase.CancellationTokenSource); + if (testCase.TestMethod is not null) + { + TrySet(displayNameSetter, instance, testCase.DisplayName); + if (testCase.Arguments is not null) + TrySet(dataSetter, instance, testCase.Arguments); + } + + return instance; + } + + private Dictionary BuildProperties(TestCaseContext testCase) + { + var properties = new Dictionary(StringComparer.Ordinal); + void Put(string name, object? value) => properties[labels.GetValueOrDefault(name, name)] = value; + + Put("FullyQualifiedTestClassName", testCase.TestClass.FullName); + Put("ManagedType", testCase.TestClass.FullName); + if (testCase.TestMethod is not null) + { + Put("TestName", testCase.TestMethod.Name); + Put("ManagedMethod", testCase.TestMethod.Name); + } + + // Nothing is deployed and there is no results folder of MSTest's own, so every directory + // MSTest would name is the one the test assembly was loaded from, which always exists. + var directory = Path.GetDirectoryName(testCase.TestClass.Assembly.Location); + if (!string.IsNullOrEmpty(directory)) + { + foreach (var name in (string[])["TestRunDirectory", "DeploymentDirectory", "ResultsDirectory", + "TestRunResultsDirectory", "TestResultsDirectory", "TestDir", "TestDeploymentDir", "TestLogsDir"]) + { + Put(name, directory); + } + } + + return properties; + } + + private static void TrySet(MethodInfo? setter, object instance, object? value) + { + if (setter is null) + return; + + try { setter.Invoke(instance, [value]); } + catch { /* optional detail; the context works without it */ } + } + } +} diff --git a/src/Motus.Runner/Services/Execution/TestCaseContext.cs b/src/Motus.Runner/Services/Execution/TestCaseContext.cs new file mode 100644 index 0000000..b15fcde --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestCaseContext.cs @@ -0,0 +1,53 @@ +using System.Reflection; + +namespace Motus.Runner.Services.Execution; + +/// +/// The outcome a running test reports through MSTest's TestContext.CurrentTestOutcome. +/// +internal enum TestCaseOutcome +{ + InProgress, + Passed, + Failed, + Inconclusive, +} + +/// +/// What the runners know about the test being run, from which an MSTest TestContext is +/// built. One is made per test case, and one for each [AssemblyInitialize] and +/// [ClassInitialize] call. +/// +internal sealed class TestCaseContext +{ + public TestCaseContext(Type testClass, MethodInfo? testMethod, string? displayName = null, object?[]? arguments = null) + { + TestClass = testClass; + TestMethod = testMethod; + DisplayName = displayName ?? testMethod?.Name; + Arguments = arguments; + } + + public Type TestClass { get; } + + /// The test method, or null for an assembly or class initializer. + public MethodInfo? TestMethod { get; } + + /// The name of this test case, including its data row's arguments. + public string? DisplayName { get; } + + /// The data row this test case runs with, or null for a test without one. + public object?[]? Arguments { get; } + + /// + /// while the test is set up and run, then the result + /// of the test itself, which is what the test's cleanup methods see. + /// + public TestCaseOutcome Outcome { get; set; } = TestCaseOutcome.InProgress; + + /// + /// Cancelled when the run is stopped. Not disposed, because a test may keep its + /// TestContext in a static field and read the token after it has finished. + /// + public CancellationTokenSource CancellationTokenSource { get; } = new(); +} diff --git a/src/Motus.Runner/Services/Execution/TestCaseExpander.cs b/src/Motus.Runner/Services/Execution/TestCaseExpander.cs new file mode 100644 index 0000000..b276320 --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestCaseExpander.cs @@ -0,0 +1,131 @@ +using System.Collections; +using System.Globalization; +using System.Reflection; + +namespace Motus.Runner.Services.Execution; + +/// +/// One test case found by discovery: a test method, or one data row of a data-driven one. +/// +/// The name the runners report and filter on. +/// The data row, or null for a test method without one. +/// Set when the data row itself is marked ignored. +internal sealed record ExpandedTestCase(string FullName, object?[]? Arguments, string? IgnoreReason); + +/// +/// Turns a test method into its test cases, one per [DataRow], [DynamicData] row or +/// other MSTest ITestDataSource row, the way MSTest does. Shared by both runners' discovery. +/// +internal static class TestCaseExpander +{ + private const string TestDataSourceInterfaceName = "Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSource"; + + /// + /// True when 's type, or a type it derives from, is named in + /// , so that [DataTestMethod] and other subclasses of + /// [TestMethod] or [TestClass] count as what they extend. + /// + public static bool IsOrDerivesFrom(object attribute, IReadOnlySet names) + { + for (var type = attribute.GetType(); type is not null && type != typeof(Attribute); type = type.BaseType) + { + if (names.Contains(type.Name)) + return true; + } + + return false; + } + + /// + /// The test cases of on : a single case + /// named {Class}.{Method} for a method without data rows, or one per row named + /// {Class}.{display name}, where the display name is MSTest's own, such as + /// SignsIn ("owner"). is told about a data source that + /// could not be read. + /// + public static List Expand(Type testClass, MethodInfo method, Action? onError = null) + { + var baseName = $"{testClass.FullName}.{method.Name}"; + var cases = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var attribute in method.GetCustomAttributes(true)) + { + var dataSource = attribute.GetType().GetInterfaces().FirstOrDefault(i => i.FullName == TestDataSourceInterfaceName); + if (dataSource is null) + continue; + + List rows; + try + { + var data = dataSource.GetMethod("GetData")!.Invoke(attribute, [method]) as IEnumerable; + rows = data?.Cast().ToList() ?? []; + } + catch (Exception ex) + { + var inner = ex is TargetInvocationException { InnerException: { } e } ? e : ex; + onError?.Invoke($"Could not read the test data for {baseName} from [{attribute.GetType().Name}]: {inner.Message}"); + continue; + } + + // MSTest 3.8 and later can ignore a single row. + var ignoreReason = attribute.GetType().GetProperty("IgnoreMessage")?.GetValue(attribute) as string; + var getDisplayName = dataSource.GetMethod("GetDisplayName"); + + foreach (var row in rows) + { + var displayName = TryGetDisplayName(getDisplayName, attribute, method, row) + ?? DefaultDisplayName(method, row); + + // A display name that drops the method name still gets it, so --filter by method works. + var fullName = displayName.StartsWith(method.Name, StringComparison.Ordinal) + ? $"{testClass.FullName}.{displayName}" + : $"{baseName} ({displayName})"; + + // Two rows with the same values would otherwise share a name, and the runners key results by it. + var unique = fullName; + for (var n = 2; !seen.Add(unique); n++) + unique = $"{fullName} #{n}"; + + cases.Add(new ExpandedTestCase(unique, row, ignoreReason)); + } + } + + // No data source, or none that could be read: a single case. One whose method takes + // parameters fails when run, saying they were not supplied. + if (cases.Count == 0) + cases.Add(new ExpandedTestCase(baseName, null, null)); + + return cases; + } + + private static string? TryGetDisplayName(MethodInfo? getDisplayName, object attribute, MethodInfo method, object?[] row) + { + if (getDisplayName is null) + return null; + + try + { + var name = getDisplayName.Invoke(attribute, [method, row]) as string; + return string.IsNullOrWhiteSpace(name) ? null : name; + } + catch + { + return null; + } + } + + private static string DefaultDisplayName(MethodInfo method, object?[] row) + => $"{method.Name} ({string.Join(", ", row.Select(FormatValue))})"; + + private static string FormatValue(object? value) => value switch + { + null => "null", + string s => $"\"{s}\"", + char c => $"'{c}'", + bool b => b ? "true" : "false", + IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), + Array a => $"[{string.Join(", ", a.Cast().Select(FormatValue))}]", + _ => value.ToString() ?? value.GetType().Name, + }; +} diff --git a/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs b/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs new file mode 100644 index 0000000..a6749df --- /dev/null +++ b/src/Motus.Runner/Services/Execution/TestCaseInvoker.cs @@ -0,0 +1,279 @@ +using System.Diagnostics; +using System.Reflection; +using Motus; + +namespace Motus.Runner.Services.Execution; + +/// +/// The outcome of one test case run by . +/// +/// The exception that failed the test, or null when it passed. +/// The time from creating the test instance to the end of the test method, +/// not counting cleanup. +internal sealed record TestCaseRunResult(Exception? Failure, TimeSpan Duration); + +/// +/// Runs one test case by reflection the way its framework would: creates the instance, gives an +/// MSTest test its TestContext, runs setup base class first, runs the test with its data +/// row, then runs cleanup derived class first. Shared by motus run and the visual runner. +/// +internal static class TestCaseInvoker +{ + /// + /// Runs on a new instance of . + /// + /// The class to create the test instance from. + /// The test method. + /// The data row to call it with, or null for a test without one. + /// The test case's name, reported as MSTest's TestDisplayName. + /// Runs first, inside the test, so a failure fails the test. Used for + /// [ClassInitialize]. + /// Told about each cleanup method or Dispose that throws. + /// Cleanup failures do not fail the test. + /// Cancels the token MSTest tests read from their TestContext. + public static async Task RunAsync( + Type testClass, + MethodInfo testMethod, + object?[]? arguments, + string? displayName, + Func? beforeInstance = null, + Action? onCleanupFailure = null, + CancellationToken cancellationToken = default) + { + var testCase = new TestCaseContext(testClass, testMethod, displayName, arguments); + using var cancellation = cancellationToken.Register(() => testCase.CancellationTokenSource.Cancel()); + + var sw = Stopwatch.StartNew(); + object? instance = null; + object? context = null; + Type? contextType = null; + Exception? failure = null; + + // Publish the test method name so MotusTestBase can resolve per-method attributes + // (e.g. [PerformanceBudget]) and name failure artifacts without a TestContext + TestMethodNameContext.Set(testMethod.Name); + try + { + if (beforeInstance is not null) + await beforeInstance(); + + contextType = FindTestContextType(testClass); + if (contextType is not null) + context = MSTestContextFactory.TryCreate(contextType, testCase); + + instance = CreateInstance(testClass, context); + if (context is not null) + { + AssignTestContext(instance, context); + MSTestContextFactory.SetCurrent(contextType!, context); + } + + // [TestInitialize] / NUnit [SetUp], base class first + await RunInstanceHooksAsync(instance, testClass, "TestInitializeAttribute", baseFirst: true); + await RunInstanceHooksAsync(instance, testClass, "SetUpAttribute", baseFirst: true); + + await InvokeAsync(testMethod, instance, BindArguments(testMethod, arguments)); + + testCase.Outcome = TestCaseOutcome.Passed; + } + catch (Exception ex) + { + failure = Unwrap(ex); + testCase.Outcome = failure.GetType().Name == "AssertInconclusiveException" + ? TestCaseOutcome.Inconclusive + : TestCaseOutcome.Failed; + if (context is not null) + MSTestContextFactory.SetException(context, failure); + } + finally + { + sw.Stop(); + + if (instance is not null) + { + // [TestCleanup] / NUnit [TearDown], derived class first so that a derived cleanup + // still has what the base class set up. Each one runs even if an earlier one threw, + // so a base class that releases a browser context always gets to. + await RunCleanupHooksAsync(instance, testClass, "TestCleanupAttribute", onCleanupFailure); + await RunCleanupHooksAsync(instance, testClass, "TearDownAttribute", onCleanupFailure); + + try + { + if (instance is IAsyncDisposable asyncDisposable) + await asyncDisposable.DisposeAsync(); + else if (instance is IDisposable disposable) + disposable.Dispose(); + } + catch (Exception ex) + { + onCleanupFailure?.Invoke(Unwrap(ex)); + } + } + + if (context is not null) + MSTestContextFactory.SetCurrent(contextType!, null); + TestMethodNameContext.Clear(); + } + + return new TestCaseRunResult(failure, sw.Elapsed); + } + + /// + /// Runs a static lifecycle method such as [AssemblyInitialize] or [ClassInitialize], + /// handing an MSTest one the TestContext it takes. is the + /// class the context describes. + /// + public static Task InvokeStaticAsync(MethodInfo method, Type testClass) + { + var parameters = method.GetParameters(); + var args = new object?[parameters.Length]; + if (parameters.Length > 0) + { + var contextType = parameters[0].ParameterType; + if (MSTestContextFactory.IsTestContextType(contextType)) + args[0] = MSTestContextFactory.TryCreate(contextType, new TestCaseContext(testClass, testMethod: null)); + } + + return InvokeAsync(method, null, args); + } + + /// + /// The methods carrying on and its + /// base classes: base class first when , derived class first otherwise. + /// + internal static List GetInstanceHooks(Type testClass, string attributeName, bool baseFirst) + { + // GetMethods with FlattenHierarchy does not guarantee ordering, so walk the hierarchy. + var levels = new List>(); + for (var type = testClass; type is not null && type != typeof(object); type = type.BaseType) + { + levels.Add(type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) + .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName))); + } + + if (baseFirst) + levels.Reverse(); + + return levels.SelectMany(level => level).ToList(); + } + + /// + /// Fits a data row to 's parameters: fills omitted optional parameters + /// with their defaults and gathers trailing values into a params array. + /// + internal static object?[]? BindArguments(MethodInfo method, object?[]? arguments) + { + var parameters = method.GetParameters(); + if (parameters.Length == 0) + return arguments is { Length: > 0 } ? arguments : null; + + arguments ??= []; + + var last = parameters[^1]; + if (last.ParameterType.IsArray && last.IsDefined(typeof(ParamArrayAttribute)) && arguments.Length >= parameters.Length - 1) + { + var alreadyAnArray = arguments.Length == parameters.Length + && (arguments[^1] is null || last.ParameterType.IsInstanceOfType(arguments[^1])); + if (!alreadyAnArray) + { + var fixedCount = parameters.Length - 1; + var rest = Array.CreateInstance(last.ParameterType.GetElementType()!, arguments.Length - fixedCount); + for (var i = 0; i < rest.Length; i++) + rest.SetValue(arguments[fixedCount + i], i); + arguments = [.. arguments.Take(fixedCount), rest]; + } + } + + if (arguments.Length < parameters.Length && parameters.Skip(arguments.Length).All(p => p.IsOptional)) + arguments = [.. arguments, .. Enumerable.Repeat(Type.Missing, parameters.Length - arguments.Length)]; + + if (arguments.Length != parameters.Length) + { + throw new InvalidOperationException( + $"Test method {method.DeclaringType?.Name}.{method.Name} takes {parameters.Length} parameter(s) but was given " + + $"{arguments.Length}. Supply them with [DataRow] or [DynamicData]."); + } + + return arguments; + } + + private static Type? FindTestContextType(Type testClass) + { + foreach (var property in testClass.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (property.Name == "TestContext" && MSTestContextFactory.IsTestContextType(property.PropertyType)) + return property.PropertyType; + } + + foreach (var constructor in testClass.GetConstructors()) + { + var parameters = constructor.GetParameters(); + if (parameters.Length == 1 && MSTestContextFactory.IsTestContextType(parameters[0].ParameterType)) + return parameters[0].ParameterType; + } + + return null; + } + + private static object CreateInstance(Type testClass, object? context) + { + // MSTest 3.6 and later can take the TestContext as a constructor parameter instead. + if (context is not null && testClass.GetConstructor(Type.EmptyTypes) is null) + { + var constructor = testClass.GetConstructors().FirstOrDefault(c => + c.GetParameters() is [var only] && only.ParameterType.IsInstanceOfType(context)); + if (constructor is not null) + return constructor.Invoke([context]); + } + + return Activator.CreateInstance(testClass)!; + } + + private static void AssignTestContext(object instance, object context) + { + foreach (var property in instance.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (property.Name == "TestContext" + && property.PropertyType.IsInstanceOfType(context) + && property.GetSetMethod() is { } setter) + { + setter.Invoke(instance, [context]); + } + } + } + + private static async Task RunInstanceHooksAsync(object instance, Type testClass, string attributeName, bool baseFirst) + { + foreach (var method in GetInstanceHooks(testClass, attributeName, baseFirst)) + await InvokeAsync(method, instance, null); + } + + private static async Task RunCleanupHooksAsync(object instance, Type testClass, string attributeName, Action? onFailure) + { + foreach (var method in GetInstanceHooks(testClass, attributeName, baseFirst: false)) + { + try + { + await InvokeAsync(method, instance, null); + } + catch (Exception ex) + { + onFailure?.Invoke(Unwrap(ex)); + } + } + } + + private static async Task InvokeAsync(MethodInfo method, object? instance, object?[]? args) + { + var result = method.Invoke(instance, args); + if (result is Task task) + await task; + } + + private static Exception Unwrap(Exception ex) + { + while (ex is TargetInvocationException { InnerException: { } inner }) + ex = inner; + return ex; + } +} diff --git a/src/Motus.Runner/Services/Models/DiscoveredTest.cs b/src/Motus.Runner/Services/Models/DiscoveredTest.cs index b94c1e7..4737ec4 100644 --- a/src/Motus.Runner/Services/Models/DiscoveredTest.cs +++ b/src/Motus.Runner/Services/Models/DiscoveredTest.cs @@ -11,4 +11,5 @@ public sealed record DiscoveredTest( string? IgnoreReason = null, string? CodeBase = null, IReadOnlyList? Categories = null, - string? ClassFullName = null); + string? ClassFullName = null, + object?[]? Arguments = null); diff --git a/src/Motus.Runner/Services/TestDiscovery.cs b/src/Motus.Runner/Services/TestDiscovery.cs index 07d823e..d77ccdb 100644 --- a/src/Motus.Runner/Services/TestDiscovery.cs +++ b/src/Motus.Runner/Services/TestDiscovery.cs @@ -1,4 +1,5 @@ using System.Reflection; +using Motus.Runner.Services.Execution; using Motus.Runner.Services.Models; namespace Motus.Runner.Services; @@ -18,6 +19,12 @@ public sealed class TestDiscovery(ILogger logger) "FactAttribute", }; + // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply. + private static readonly HashSet ExcludedMethodAttributes = new(StringComparer.Ordinal) + { + "TheoryAttribute", + }; + private static readonly HashSet IgnoreAttributes = new(StringComparer.Ordinal) { "IgnoreAttribute", // MSTest + NUnit @@ -55,15 +62,18 @@ public List Discover(string[] assemblyPaths, string? filter) if (!IsTestMethod(method)) continue; - var fullName = $"{type.FullName}.{method.Name}"; + var (methodIgnored, methodReason) = GetIgnoreInfo(method.GetCustomAttributes(true)); - if (filter is not null && !fullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) - continue; + foreach (var testCase in TestCaseExpander.Expand(type, method, message => logger.LogWarning("{Message}", message))) + { + if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase)) + continue; - var (methodIgnored, methodReason) = GetIgnoreInfo(method.GetCustomAttributes(true)); - var isIgnored = classIgnored || methodIgnored; - var reason = classIgnored ? classReason : methodReason; - tests.Add(new DiscoveredTest(type, method, fullName, assemblyName, isIgnored, reason)); + var isIgnored = classIgnored || methodIgnored || testCase.IgnoreReason is not null; + var reason = classIgnored ? classReason : methodIgnored ? methodReason : testCase.IgnoreReason; + tests.Add(new DiscoveredTest(type, method, testCase.FullName, assemblyName, isIgnored, reason, + Arguments: testCase.Arguments)); + } } } } @@ -77,13 +87,14 @@ private static bool IsTestClass(Type type) return false; return type.GetCustomAttributes(true) - .Any(a => ClassAttributes.Contains(a.GetType().Name)); + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes)); } private static bool IsTestMethod(MethodInfo method) { return method.GetCustomAttributes(true) - .Any(a => MethodAttributes.Contains(a.GetType().Name)); + .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes) + && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes)); } private static (bool IsIgnored, string? Reason) GetIgnoreInfo(object[] attrs) diff --git a/src/Motus.Runner/Services/TestExecutionService.cs b/src/Motus.Runner/Services/TestExecutionService.cs index 0bddead..944c244 100644 --- a/src/Motus.Runner/Services/TestExecutionService.cs +++ b/src/Motus.Runner/Services/TestExecutionService.cs @@ -4,6 +4,7 @@ using Motus; using Motus.Abstractions; using Motus.Runner.Services.Coverage; +using Motus.Runner.Services.Execution; using Motus.Runner.Services.Models; namespace Motus.Runner.Services; @@ -17,6 +18,9 @@ public sealed class TestExecutionService(ILogger logger, I // Track which assemblies have had [AssemblyInitialize] run private readonly ConcurrentDictionary _initializedAssemblies = new(); + // Track which classes have had [ClassInitialize] / [OneTimeSetUp] run in this batch + private readonly ConcurrentDictionary _initializedClasses = new(); + public async Task ExecuteAsync( List tests, Action onStateChanged, @@ -88,7 +92,7 @@ public async Task ExecuteAsync( await reporters.FireOnTestStartAsync(new TestInfo(test.FullName, suiteName)); CoverageSink.Begin(); - var result = await ExecuteTestAsync(test, logger); + var result = await ExecuteTestAsync(test, ct); onStateChanged(result); var coverage = CoverageSink.End(); @@ -133,6 +137,23 @@ public async Task ExecuteAsync( logger.LogInformation("Test run cancelled"); } + // Run [ClassCleanup] / [OneTimeTearDown] for each class initialized in this batch + foreach (var type in _initializedClasses.Keys.ToList()) + { + if (_initializedClasses.TryRemove(type, out _)) + { + try + { + await RunStaticLifecycleMethodsAsync(type, "ClassCleanupAttribute"); + await RunStaticLifecycleMethodsAsync(type, "OneTimeTearDownAttribute"); + } + catch (Exception ex) + { + logger.LogWarning(ex, "ClassCleanup failed for {Class}", type.FullName); + } + } + } + // Run [AssemblyCleanup] for each initialized assembly to close browsers foreach (var assembly in assemblies) { @@ -174,30 +195,6 @@ await reporters.FireOnTestRunEndAsync( } } - private static async Task ExecuteLifecycleMethodAsync(object instance, Type testClass, string attributeName) - { - // Walk the hierarchy bottom-up, then reverse so base-class methods run first. - // GetMethods with FlattenHierarchy does not guarantee ordering, and running a - // derived [TestInitialize] before the base (e.g. MotusTestBase.MotusTestInitialize) - // causes null-reference failures when the derived method accesses Page/Context. - var methods = new List(); - var type = testClass; - while (type is not null && type != typeof(object)) - { - var declared = type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly) - .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); - methods.InsertRange(0, declared); - type = type.BaseType; - } - - foreach (var method in methods) - { - var result = method.Invoke(instance, null); - if (result is Task task) - await task; - } - } - private static async Task RunAssemblyInitializeAsync(Assembly assembly) { // [AssemblyInitialize] is a static method that takes TestContext @@ -208,14 +205,7 @@ private static async Task RunAssemblyInitializeAsync(Assembly assembly) .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyInitializeAttribute")); foreach (var method in methods) - { - var parameters = method.GetParameters(); - // MSTest [AssemblyInitialize] takes a TestContext parameter - object?[] args = parameters.Length > 0 ? [null] : []; - var result = method.Invoke(null, args); - if (result is Task task) - await task; - } + await TestCaseInvoker.InvokeStaticAsync(method, type); } } @@ -227,72 +217,48 @@ private static async Task RunAssemblyCleanupAsync(Assembly assembly) .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyCleanupAttribute")); foreach (var method in methods) - { - var result = method.Invoke(null, null); - if (result is Task task) - await task; - } + await TestCaseInvoker.InvokeStaticAsync(method, type); } } - private static async Task ExecuteTestAsync(DiscoveredTest test, ILogger logger) + private static async Task RunStaticLifecycleMethodsAsync(Type type, string attributeName) { - var sw = Stopwatch.StartNew(); - object? instance = null; - try - { - instance = Activator.CreateInstance(test.TestClass!)!; - - // Publish the test method name so MotusTestBase can resolve - // per-method attributes (e.g. [PerformanceBudget]) without TestContext - TestMethodNameContext.Set(test.TestMethod!.Name); - - // Run [TestInitialize] methods - await ExecuteLifecycleMethodAsync(instance, test.TestClass!, "TestInitializeAttribute"); - // Also support NUnit [SetUp] - await ExecuteLifecycleMethodAsync(instance, test.TestClass!, "SetUpAttribute"); - - var result = test.TestMethod!.Invoke(instance, null); + var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy) + .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName)); - if (result is Task task) - await task; + foreach (var method in methods) + await TestCaseInvoker.InvokeStaticAsync(method, type); + } - sw.Stop(); - return new TestNodeState(test.FullName, TestStatus.Passed, sw.Elapsed, null, null); - } - catch (TargetInvocationException ex) when (ex.InnerException is not null) - { - sw.Stop(); - return new TestNodeState(test.FullName, TestStatus.Failed, sw.Elapsed, ex.InnerException.Message, ex.InnerException.StackTrace); - } - catch (Exception ex) - { - sw.Stop(); - return new TestNodeState(test.FullName, TestStatus.Failed, sw.Elapsed, ex.Message, ex.StackTrace); - } - finally - { - if (instance is not null) + private async Task ExecuteTestAsync(DiscoveredTest test, CancellationToken ct) + { + var testClass = test.TestClass!; + var run = await TestCaseInvoker.RunAsync( + testClass, + test.TestMethod!, + test.Arguments, + DisplayNameOf(test), + beforeInstance: async () => { - try + // Run [ClassInitialize] / [OneTimeSetUp] once per class, before its first test + if (_initializedClasses.TryAdd(testClass, true)) { - // Run [TestCleanup] methods - await ExecuteLifecycleMethodAsync(instance, test.TestClass!, "TestCleanupAttribute"); - // Also support NUnit [TearDown] - await ExecuteLifecycleMethodAsync(instance, test.TestClass!, "TearDownAttribute"); - } - catch (Exception ex) - { - logger.LogWarning(ex, "TestCleanup failed for {Test}", test.FullName); + await RunStaticLifecycleMethodsAsync(testClass, "ClassInitializeAttribute"); + await RunStaticLifecycleMethodsAsync(testClass, "OneTimeSetUpAttribute"); } + }, + onCleanupFailure: ex => logger.LogWarning(ex, "TestCleanup failed for {Test}", test.FullName), + cancellationToken: ct); - if (instance is IAsyncDisposable asyncDisposable) - await asyncDisposable.DisposeAsync(); - else if (instance is IDisposable disposable) - disposable.Dispose(); - } + return run.Failure is null + ? new TestNodeState(test.FullName, TestStatus.Passed, run.Duration, null, null) + : new TestNodeState(test.FullName, TestStatus.Failed, run.Duration, run.Failure.Message, run.Failure.StackTrace); + } - TestMethodNameContext.Clear(); - } + /// The test case's name without its class, such as SignsIn ("owner"). + private static string DisplayNameOf(DiscoveredTest test) + { + var prefix = test.TestClass!.FullName + "."; + return test.FullName.StartsWith(prefix, StringComparison.Ordinal) ? test.FullName[prefix.Length..] : test.TestMethod!.Name; } } diff --git a/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj b/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj index 93b6db4..72708bf 100644 --- a/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj +++ b/tests/Motus.Cli.Tests/Motus.Cli.Tests.csproj @@ -23,6 +23,7 @@ + diff --git a/tests/Motus.Cli.Tests/Services/FailureTraceIntegrationTests.cs b/tests/Motus.Cli.Tests/Services/FailureTraceIntegrationTests.cs new file mode 100644 index 0000000..702a4ae --- /dev/null +++ b/tests/Motus.Cli.Tests/Services/FailureTraceIntegrationTests.cs @@ -0,0 +1,115 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Motus.Abstractions; +using Motus.Cli.Services; +using Motus.Cli.Services.Reporters; +using Motus.Testing.MSTest; + +namespace Motus.Cli.Tests.Services; + +/// +/// Runs tests through motus run with failure tracing on, and +/// checks that only the failing test leaves a trace behind. +/// +[TestClass] +[TestCategory("Integration")] +public class FailureTraceIntegrationTests +{ + // Not a [TestClass]: run only through TestRunner. + public class TracedFixture : MotusTestBase + { + public async Task Passes() => await Page.GotoAsync("data:text/html,

passes

"); + + public async Task Fails() + { + await Page.GotoAsync("data:text/html,

fails

"); + throw new InvalidOperationException("intentional failure"); + } + } + + private static bool s_browserAvailable; + + [ClassInitialize] + public static async Task ClassSetup(TestContext _) + { + try + { + var probe = await MotusLauncher.LaunchAsync(new LaunchOptions { Headless = true }); + await probe.CloseAsync(); + s_browserAvailable = true; + } + catch (FileNotFoundException) + { + s_browserAvailable = false; + } + } + + [ClassCleanup] + public static void ClassTeardown() + { + try + { + if (MotusConfigLoader.Config.Failure?.TracePath is { } directory) + Directory.Delete(directory, recursive: true); + } + catch { } + } + + [TestInitialize] + public void Setup() + { + if (!s_browserAvailable) + Assert.Inconclusive("No browser found; skipping integration tests."); + + Assert.IsTrue(MotusConfigLoader.Config.Failure?.Trace is true, "failure tracing should be on for this assembly"); + } + + [TestMethod] + public async Task PassingTest_SavesNoTrace_FailingTest_SavesOne() + { + var traceDirectory = MotusConfigLoader.Config.Failure!.TracePath!; + + var result = await new TestRunner(1).RunAsync( + [MakeTest(nameof(TracedFixture.Passes)), MakeTest(nameof(TracedFixture.Fails))], + new ConsoleReporter()); + + Assert.AreEqual(1, result.Passed); + Assert.AreEqual(1, result.Failed); + Assert.AreEqual(0, TracesFor(traceDirectory, nameof(TracedFixture.Passes)).Length, + "a passing test should leave no failure trace"); + Assert.AreEqual(1, TracesFor(traceDirectory, nameof(TracedFixture.Fails)).Length, + "a failing test should still leave its failure trace"); + } + + private static DiscoveredTest MakeTest(string methodName) + { + var type = typeof(TracedFixture); + var method = type.GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance)!; + return new DiscoveredTest(type, method, $"{type.FullName}.{methodName}", false); + } + + private static string[] TracesFor(string directory, string methodName) + => Directory.Exists(directory) + ? Directory.GetFiles(directory, $"trace-{nameof(TracedFixture)}.{methodName}-*.zip") + : []; +} + +internal static class FailureTracingSetup +{ + /// + /// Failure tracing is read from configuration once per process, so it is switched on as this + /// assembly loads, before anything reads it, with traces going to a folder of this run's own. + /// Only acts on it, and only + /// uses that here. + /// +#pragma warning disable CA2255 // a test assembly, not a library + [ModuleInitializer] +#pragma warning restore CA2255 + internal static void EnableFailureTracing() + { + Environment.SetEnvironmentVariable("MOTUS_FAILURES_TRACE", "true"); + Environment.SetEnvironmentVariable( + "MOTUS_FAILURES_TRACE_PATH", + Path.Combine(Path.GetTempPath(), $"motus-cli-tests-traces-{Guid.NewGuid():N}")); + } +} diff --git a/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs b/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs new file mode 100644 index 0000000..f7f3f92 --- /dev/null +++ b/tests/Motus.Cli.Tests/Services/TestRunnerMSTestTests.cs @@ -0,0 +1,339 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Motus.Abstractions; +using Motus.Cli.Commands; +using Motus.Cli.Services; + +namespace Motus.Cli.Tests.Services; + +/// +/// MSTest features that motus run has to provide the way MSTest's own adapter does, so that a +/// suite that runs under dotnet test runs unchanged under the runner. +/// +[TestClass] +public class TestRunnerMSTestTests +{ + #region Fixtures (not [TestClass]: run only through TestRunner) + + public class ContextFixture + { + public static List Log { get; } = []; + + public TestContext TestContext { get; set; } = null!; + + [TestInitialize] + public void Init() => Log.Add($"init {TestContext?.TestName} {TestContext?.CurrentTestOutcome}"); + + // The check MotusTestBase makes before it decides whether to keep a failure trace. + [TestCleanup] + public void Cleanup() => Log.Add($"cleanup {TestContext?.CurrentTestOutcome} saveTrace={TestContext?.CurrentTestOutcome != UnitTestOutcome.Passed}"); + + public void Passes() + { + TestContext.WriteLine("hello from {0}", TestContext.TestName); + Log.Add($"class {TestContext.FullyQualifiedTestClassName}"); + Log.Add($"token {TestContext.CancellationTokenSource is not null}"); + } + + public void Fails() => throw new InvalidOperationException("intentional failure"); + + public void IsInconclusive() => Assert.Inconclusive("not decided"); + } + + public class ConstructorContextFixture(TestContext testContext) + { + public static string? SeenName { get; set; } + + public void UsesContext() => SeenName = testContext.TestName; + } + + public class ClassInitializeFixture + { + public static TestContext? Received { get; set; } + + [ClassInitialize] + public static void Setup(TestContext context) => Received = context; + + public void Test() { } + } + + public abstract class OrderBase + { + public static List Log { get; } = []; + + [TestInitialize] + public void BaseInit() => Log.Add("base init"); + + [TestCleanup] + public void BaseCleanup() => Log.Add("base cleanup"); + } + + public class OrderFixture : OrderBase + { + [TestInitialize] + public void DerivedInit() => Log.Add("derived init"); + + [TestCleanup] + public void DerivedCleanup() => Log.Add("derived cleanup"); + + public void Test() => Log.Add("test"); + } + + public abstract class NUnitOrderBase + { + public static List Log { get; } = []; + + [SetUp] + public void BaseSetUp() => Log.Add("base setup"); + + [TearDown] + public void BaseTearDown() => Log.Add("base teardown"); + } + + public class NUnitOrderFixture : NUnitOrderBase + { + [SetUp] + public void DerivedSetUp() => Log.Add("derived setup"); + + [TearDown] + public void DerivedTearDown() => Log.Add("derived teardown"); + + public void Test() => Log.Add("test"); + } + + public abstract class ReleasingBase + { + public static bool Released { get; set; } + + [TestCleanup] + public void Release() => Released = true; + } + + public class ThrowingCleanupFixture : ReleasingBase + { + [TestCleanup] + public void Throws() => throw new InvalidOperationException("derived cleanup failed"); + + public void Test() { } + } + + // The runners find NUnit's lifecycle methods by attribute name, so these stand in for NUnit's own. + [AttributeUsage(AttributeTargets.Method)] + public sealed class SetUpAttribute : Attribute; + + [AttributeUsage(AttributeTargets.Method)] + public sealed class TearDownAttribute : Attribute; + + #endregion + + private static NullReporter Reporter { get; } = new(); + + private static DiscoveredTest MakeTest(Type type, string methodName) + { + var method = type.GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance)!; + return new DiscoveredTest(type, method, $"{type.FullName}.{methodName}", false); + } + + private static Task RunAsync(params DiscoveredTest[] tests) + => new TestRunner(1).RunAsync([.. tests], Reporter); + + [TestMethod] + public async Task TestContext_IsSetBeforeTestInitialize_AndDescribesTheTest() + { + ContextFixture.Log.Clear(); + + var result = await RunAsync(MakeTest(typeof(ContextFixture), nameof(ContextFixture.Passes))); + + Assert.AreEqual(1, result.Passed); + CollectionAssert.Contains(ContextFixture.Log, $"init {nameof(ContextFixture.Passes)} InProgress"); + CollectionAssert.Contains(ContextFixture.Log, $"class {typeof(ContextFixture).FullName}"); + CollectionAssert.Contains(ContextFixture.Log, "token True"); + } + + [TestMethod] + public async Task PassingTest_CleanupSeesPassed_SoNoFailureTraceIsKept() + { + ContextFixture.Log.Clear(); + + await RunAsync(MakeTest(typeof(ContextFixture), nameof(ContextFixture.Passes))); + + CollectionAssert.Contains(ContextFixture.Log, "cleanup Passed saveTrace=False"); + } + + [TestMethod] + public async Task FailingTest_CleanupSeesFailed_SoTheFailureTraceIsKept() + { + ContextFixture.Log.Clear(); + + var result = await RunAsync(MakeTest(typeof(ContextFixture), nameof(ContextFixture.Fails))); + + Assert.AreEqual(1, result.Failed); + CollectionAssert.Contains(ContextFixture.Log, "cleanup Failed saveTrace=True"); + } + + [TestMethod] + public async Task InconclusiveTest_CleanupSeesInconclusive() + { + ContextFixture.Log.Clear(); + + await RunAsync(MakeTest(typeof(ContextFixture), nameof(ContextFixture.IsInconclusive))); + + CollectionAssert.Contains(ContextFixture.Log, "cleanup Inconclusive saveTrace=True"); + } + + [TestMethod] + public async Task TestContext_CanBeTakenByTheConstructor() + { + ConstructorContextFixture.SeenName = null; + + var result = await RunAsync(MakeTest(typeof(ConstructorContextFixture), nameof(ConstructorContextFixture.UsesContext))); + + Assert.AreEqual(1, result.Passed); + Assert.AreEqual(nameof(ConstructorContextFixture.UsesContext), ConstructorContextFixture.SeenName); + } + + [TestMethod] + public async Task ClassInitialize_ReceivesATestContextForItsClass() + { + ClassInitializeFixture.Received = null; + + await RunAsync(MakeTest(typeof(ClassInitializeFixture), nameof(ClassInitializeFixture.Test))); + + Assert.IsNotNull(ClassInitializeFixture.Received); + Assert.AreEqual(typeof(ClassInitializeFixture).FullName, ClassInitializeFixture.Received.FullyQualifiedTestClassName); + } + + [TestMethod] + public async Task AssemblyInitialize_ReceivesATestContext() + { + AssemblyInitializeFixture.Received = null; + + await RunAsync(MakeTest(typeof(OrderFixture), nameof(OrderFixture.Test))); + + Assert.IsNotNull(AssemblyInitializeFixture.Received); + } + + [TestMethod] + public async Task MSTestCleanup_RunsDerivedFirst_InitializeRunsBaseFirst() + { + OrderBase.Log.Clear(); + + await RunAsync(MakeTest(typeof(OrderFixture), nameof(OrderFixture.Test))); + + CollectionAssert.AreEqual( + new[] { "base init", "derived init", "test", "derived cleanup", "base cleanup" }, + OrderBase.Log); + } + + [TestMethod] + public async Task NUnitTearDown_RunsDerivedFirst_SetUpRunsBaseFirst() + { + NUnitOrderBase.Log.Clear(); + + await RunAsync(MakeTest(typeof(NUnitOrderFixture), nameof(NUnitOrderFixture.Test))); + + CollectionAssert.AreEqual( + new[] { "base setup", "derived setup", "test", "derived teardown", "base teardown" }, + NUnitOrderBase.Log); + } + + [TestMethod] + public async Task ThrowingDerivedCleanup_StillRunsTheBaseCleanup() + { + ReleasingBase.Released = false; + + var result = await RunAsync(MakeTest(typeof(ThrowingCleanupFixture), nameof(ThrowingCleanupFixture.Test))); + + Assert.IsTrue(ReleasingBase.Released, "the base class's cleanup should run after a derived cleanup throws"); + Assert.AreEqual(1, result.Passed, "a cleanup failure is reported but does not fail the test"); + } + + [TestMethod] + public async Task DataRows_AreDiscoveredAsOneTestEach_AndRunWithTheirArguments() + { + DataRowFixture.Seen.Clear(); + var tests = new TestDiscovery().Discover([typeof(DataRowFixture).Assembly.Location], nameof(DataRowFixture)); + + var names = tests.Select(t => t.FullName[(typeof(DataRowFixture).FullName!.Length + 1)..]).OrderBy(n => n, StringComparer.Ordinal).ToList(); + Assert.AreEqual(5, names.Count, string.Join(" | ", names)); + Assert.AreEqual(2, names.Count(n => n.StartsWith(nameof(DataRowFixture.SignsIn) + " (", StringComparison.Ordinal)), string.Join(" | ", names)); + Assert.IsTrue(names.Any(n => n.StartsWith(nameof(DataRowFixture.Adds) + " (", StringComparison.Ordinal)), "[DataTestMethod] should be discovered"); + Assert.AreEqual(2, names.Count(n => n.StartsWith(nameof(DataRowFixture.Squares) + " (", StringComparison.Ordinal)), string.Join(" | ", names)); + Assert.AreEqual(names.Count, names.Distinct().Count(), "each row needs a name of its own"); + + var result = await new TestRunner(1).RunAsync(tests, Reporter); + + Assert.AreEqual(5, result.Passed, $"failed: {result.Failed}"); + CollectionAssert.AreEquivalent( + new[] { "role owner", "role staff", "adds 1+2", "square 2", "square 3" }, + DataRowFixture.Seen.ToArray()); + } + + [TestMethod] + public void QuarantineList_NamesOneRow_OrEveryRowByTheMethod() + { + var rows = new TestDiscovery().Discover([typeof(DataRowFixture).Assembly.Location], $"{nameof(DataRowFixture)}.{nameof(DataRowFixture.SignsIn)}"); + var owner = rows.Single(t => t.FullName.Contains("owner")); + var staff = rows.Single(t => t.FullName.Contains("staff")); + + var oneRow = new HashSet { owner.FullName }; + Assert.IsTrue(RunCommand.IsListed(owner, oneRow)); + Assert.IsFalse(RunCommand.IsListed(staff, oneRow)); + + var wholeMethod = new HashSet { $"{typeof(DataRowFixture).FullName}.{nameof(DataRowFixture.SignsIn)}" }; + Assert.IsTrue(RunCommand.IsListed(owner, wholeMethod)); + Assert.IsTrue(RunCommand.IsListed(staff, wholeMethod)); + } + + private sealed class NullReporter : IReporter + { + public Task OnTestRunStartAsync(TestSuiteInfo suite) => Task.CompletedTask; + public Task OnTestStartAsync(TestInfo test) => Task.CompletedTask; + public Task OnTestEndAsync(TestInfo test, Motus.Abstractions.TestResult result) => Task.CompletedTask; + public Task OnTestRunEndAsync(TestRunSummary summary) => Task.CompletedTask; + } +} + +/// +/// Data-driven tests that MSTest runs as well as motus run, so the two can be compared. +/// +[TestClass] +public class DataRowFixture +{ + public static ConcurrentBag Seen { get; } = []; + + public static IEnumerable SquareRows => [[2, 4], [3, 9]]; + + [TestMethod] + [DataRow("owner")] + [DataRow("staff")] + public void SignsIn(string role) => Seen.Add($"role {role}"); + + [DataTestMethod] + [DataRow(1, 2, 3)] + public void Adds(int a, int b, int sum) + { + Assert.AreEqual(sum, a + b); + Seen.Add($"adds {a}+{b}"); + } + + [TestMethod] + [DynamicData(nameof(SquareRows))] + public void Squares(int n, int square) + { + Assert.AreEqual(square, n * n); + Seen.Add($"square {n}"); + } +} + +/// +/// Not a [TestClass], so MSTest ignores it; motus run looks for [AssemblyInitialize] +/// on every exported type and runs it whenever these tests run a fixture from this assembly. +/// +public static class AssemblyInitializeFixture +{ + public static TestContext? Received { get; set; } + + [AssemblyInitialize] + public static void Initialize(TestContext context) => Received = context; +} diff --git a/tests/Motus.Tests/Motus.Tests.csproj b/tests/Motus.Tests/Motus.Tests.csproj index 11f9d68..a3460bc 100644 --- a/tests/Motus.Tests/Motus.Tests.csproj +++ b/tests/Motus.Tests/Motus.Tests.csproj @@ -21,6 +21,16 @@ + + + + + <_Parameter1>NuGetPackageRoot + <_Parameter2>$(NuGetPackageRoot) + + + diff --git a/tests/Motus.Tests/Runner/TestCaseExecutionTests.cs b/tests/Motus.Tests/Runner/TestCaseExecutionTests.cs new file mode 100644 index 0000000..2df1d9a --- /dev/null +++ b/tests/Motus.Tests/Runner/TestCaseExecutionTests.cs @@ -0,0 +1,192 @@ +using System.Reflection; +using System.Runtime.Loader; +using Motus.Runner.Services.Execution; + +namespace Motus.Tests.Runner; + +/// +/// The pieces both runners share for running MSTest tests by reflection: data row expansion, +/// argument binding and the generated TestContext. +/// +[TestClass] +public class TestCaseExecutionTests +{ + public class Methods + { + public void Optional(int a, string b = "default") { } + + public void Params(int first, params int[] rest) { } + + public void Two(int a, int b) { } + + [TestMethod] + [DataRow(1, DisplayName = "the first row")] + public void Named(int value) { } + + [TestMethod] + [DataRow(1)] + [DataRow(1)] + public void Duplicated(int value) { } + + [TestMethod] + [DynamicData("NoSuchMember")] + public void BrokenSource(int value) { } + + [TestMethod] + public void Plain() { } + } + + private static MethodInfo Method(string name) => typeof(Methods).GetMethod(name)!; + + [TestMethod] + public void BindArguments_FillsOmittedOptionalParameters() + { + var args = TestCaseInvoker.BindArguments(Method(nameof(Methods.Optional)), [1])!; + + Assert.AreEqual(2, args.Length); + Assert.AreEqual(1, args[0]); + Assert.AreSame(Type.Missing, args[1]); + } + + [TestMethod] + public void BindArguments_GathersTrailingValuesIntoParams() + { + var args = TestCaseInvoker.BindArguments(Method(nameof(Methods.Params)), [1, 2, 3])!; + + Assert.AreEqual(2, args.Length); + CollectionAssert.AreEqual(new[] { 2, 3 }, (int[])args[1]!); + } + + [TestMethod] + public void BindArguments_ParamsWithNoTrailingValues_IsAnEmptyArray() + { + var args = TestCaseInvoker.BindArguments(Method(nameof(Methods.Params)), [1])!; + + Assert.AreEqual(0, ((int[])args[1]!).Length); + } + + [TestMethod] + public void BindArguments_MissingRequiredParameters_SaysHowToSupplyThem() + { + var ex = Assert.ThrowsException( + () => TestCaseInvoker.BindArguments(Method(nameof(Methods.Two)), null)); + + StringAssert.Contains(ex.Message, "[DataRow]"); + } + + [TestMethod] + public void Expand_UsesTheRowsDisplayName_KeepingTheMethodName() + { + var cases = TestCaseExpander.Expand(typeof(Methods), Method(nameof(Methods.Named))); + + Assert.AreEqual($"{typeof(Methods).FullName}.Named (the first row)", cases.Single().FullName); + } + + [TestMethod] + public void Expand_IdenticalRows_GetDistinctNames() + { + var cases = TestCaseExpander.Expand(typeof(Methods), Method(nameof(Methods.Duplicated))); + + Assert.AreEqual(2, cases.Count); + Assert.AreNotEqual(cases[0].FullName, cases[1].FullName); + } + + [TestMethod] + public void Expand_UnreadableDataSource_ReportsIt_AndLeavesOneCase() + { + var errors = new List(); + + var cases = TestCaseExpander.Expand(typeof(Methods), Method(nameof(Methods.BrokenSource)), errors.Add); + + Assert.AreEqual(1, errors.Count); + Assert.AreEqual($"{typeof(Methods).FullName}.BrokenSource", cases.Single().FullName); + Assert.IsNull(cases.Single().Arguments); + } + + [TestMethod] + public void Expand_MethodWithoutData_IsOneCaseWithoutArguments() + { + var cases = TestCaseExpander.Expand(typeof(Methods), Method(nameof(Methods.Plain))); + + Assert.AreEqual($"{typeof(Methods).FullName}.Plain", cases.Single().FullName); + Assert.IsNull(cases.Single().Arguments); + } + + [TestMethod] + public void TestContext_ReflectsTheOutcomeAsItChanges() + { + var testCase = new TestCaseContext(typeof(Methods), Method(nameof(Methods.Plain))); + + var context = (TestContext)MSTestContextFactory.TryCreate(typeof(TestContext), testCase)!; + + Assert.AreEqual(UnitTestOutcome.InProgress, context.CurrentTestOutcome); + testCase.Outcome = TestCaseOutcome.Passed; + Assert.AreEqual(UnitTestOutcome.Passed, context.CurrentTestOutcome); + testCase.Outcome = TestCaseOutcome.Failed; + Assert.AreEqual(UnitTestOutcome.Failed, context.CurrentTestOutcome); + } + + [TestMethod] + public void TestContext_DescribesTheTest() + { + var testCase = new TestCaseContext(typeof(Methods), Method(nameof(Methods.Plain))); + + var context = (TestContext)MSTestContextFactory.TryCreate(typeof(TestContext), testCase)!; + + Assert.AreEqual(nameof(Methods.Plain), context.TestName); + Assert.AreEqual(typeof(Methods).FullName, context.FullyQualifiedTestClassName); + Assert.IsNotNull(context.CancellationTokenSource); + Assert.IsTrue(Directory.Exists(context.TestRunDirectory)); + Assert.AreSame(context.Properties, context.Properties); + + // Writes go to the console and must not throw, with or without format arguments. + context.WriteLine("plain"); + context.WriteLine("formatted {0}", 1); + context.Write("partial"); + context.AddResultFile("/nowhere/file.txt"); + } + + [TestMethod] + public void TestContext_FromMSTest4_IsBuiltToo() + { + // MSTest 4 moved TestContext to another assembly and made more of it abstract. Load it the + // way the runners meet it, from a test assembly's own folder, beside this project's MSTest 3. + var root = typeof(TestCaseExecutionTests).Assembly.GetCustomAttributes() + .FirstOrDefault(a => a.Key == "NuGetPackageRoot")?.Value ?? ""; + var package = Path.Combine(root, "mstest.testframework", "4.0.1"); + var framework = Path.Combine(package, "lib", "net8.0", "MSTest.TestFramework.dll"); + var extensions = Path.Combine(package, "buildTransitive", "net8.0", "MSTest.TestFramework.Extensions.dll"); + if (!File.Exists(framework) || !File.Exists(extensions)) + Assert.Inconclusive($"MSTest 4 was not restored to {package}."); + + var loadContext = new AssemblyLoadContext("mstest4", isCollectible: true); + loadContext.Resolving += (alc, name) => name.Name == "MSTest.TestFramework" ? alc.LoadFromAssemblyPath(framework) : null; + try + { + var contextType = loadContext.LoadFromAssemblyPath(extensions).GetType(MSTestContextFactory.TestContextTypeName, throwOnError: true)!; + Assert.AreNotEqual(typeof(TestContext).Assembly.GetName().Name, contextType.Assembly.GetName().Name); + var testCase = new TestCaseContext(typeof(Methods), Method(nameof(Methods.Plain)), "Plain (1)", [1]); + + var context = MSTestContextFactory.TryCreate(contextType, testCase); + + Assert.IsNotNull(context, "an MSTest 4 TestContext should be built"); + Assert.AreEqual(nameof(Methods.Plain), contextType.GetProperty("TestName")!.GetValue(context)); + Assert.AreEqual("Plain (1)", contextType.GetProperty("TestDisplayName")!.GetValue(context)); + testCase.Outcome = TestCaseOutcome.Passed; + Assert.AreEqual("Passed", contextType.GetProperty("CurrentTestOutcome")!.GetValue(context)!.ToString()); + contextType.GetMethod("WriteLine", [typeof(string)])!.Invoke(context, ["hello from MSTest 4"]); + } + finally + { + loadContext.Unload(); + } + } + + [TestMethod] + public void TryCreate_ForATypeThatIsNotTestContext_ReturnsNull() + { + var testCase = new TestCaseContext(typeof(Methods), null); + + Assert.IsNull(MSTestContextFactory.TryCreate(typeof(Stream), testCase)); + } +} diff --git a/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs b/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs new file mode 100644 index 0000000..5cbad02 --- /dev/null +++ b/tests/Motus.Tests/Runner/TestExecutionServiceTests.cs @@ -0,0 +1,141 @@ +using System.Reflection; +using Microsoft.Extensions.Logging.Abstractions; +using Motus.Runner.Services; +using Motus.Runner.Services.Models; + +namespace Motus.Tests.Runner; + +/// +/// The visual runner gives MSTest tests what MSTest's own adapter does: a TestContext +/// whose outcome is the test's own during cleanup, data rows, and cleanup derived class first. +/// +[TestClass] +public class TestExecutionServiceTests +{ + #region Fixtures (not [TestClass]: run only through TestExecutionService) + + public abstract class BrowserTestBase + { + public static List Log { get; } = []; + + public TestContext TestContext { get; set; } = null!; + + [TestInitialize] + public void BaseInit() => Log.Add("base init"); + + // The check MotusTestBase makes before it decides whether to keep a failure trace. + [TestCleanup] + public void BaseCleanup() => Log.Add($"base cleanup saveTrace={TestContext?.CurrentTestOutcome != UnitTestOutcome.Passed}"); + } + + public class SignInFixture : BrowserTestBase + { + public static TestContext? ClassContext { get; set; } + + [ClassInitialize] + public static void ClassSetup(TestContext context) => ClassContext = context; + + [TestCleanup] + public void DerivedCleanup() => Log.Add("derived cleanup"); + + public void UsesContext() => TestContext.WriteLine("hi"); + + public void Fails() => throw new InvalidOperationException("intentional failure"); + + public void SignsIn(string role) => Log.Add($"signs in as {role}"); + } + + #endregion + + private static TestExecutionService CreateService() => new(NullLogger.Instance); + + private static DiscoveredTest MakeTest(string methodName, string? fullName = null, object?[]? arguments = null) + { + var type = typeof(SignInFixture); + var method = type.GetMethod(methodName, BindingFlags.Public | BindingFlags.Instance)!; + return new DiscoveredTest(type, method, fullName ?? $"{type.FullName}.{methodName}", "Motus.Tests", false, + Arguments: arguments); + } + + private static async Task> RunAsync(params DiscoveredTest[] tests) + { + var states = new List(); + await CreateService().ExecuteAsync([.. tests], s => { lock (states) states.Add(s); }, CancellationToken.None); + return states.Where(s => s.Status != TestStatus.Running).ToList(); + } + + [TestMethod] + public async Task PassingTest_HasATestContext_AndCleanupSeesItPassed() + { + BrowserTestBase.Log.Clear(); + + var states = await RunAsync(MakeTest(nameof(SignInFixture.UsesContext))); + + Assert.AreEqual(TestStatus.Passed, states.Single().Status, states.Single().ErrorMessage); + CollectionAssert.AreEqual( + new[] { "base init", "derived cleanup", "base cleanup saveTrace=False" }, + BrowserTestBase.Log); + } + + [TestMethod] + public async Task FailingTest_CleanupSeesItFailed() + { + BrowserTestBase.Log.Clear(); + + var states = await RunAsync(MakeTest(nameof(SignInFixture.Fails))); + + Assert.AreEqual(TestStatus.Failed, states.Single().Status); + CollectionAssert.Contains(BrowserTestBase.Log, "base cleanup saveTrace=True"); + } + + [TestMethod] + public async Task DataRowTest_RunsWithItsArguments() + { + BrowserTestBase.Log.Clear(); + var fullName = $"{typeof(SignInFixture).FullName}.SignsIn (owner)"; + + var states = await RunAsync(MakeTest(nameof(SignInFixture.SignsIn), fullName, ["owner"])); + + Assert.AreEqual(TestStatus.Passed, states.Single().Status, states.Single().ErrorMessage); + Assert.AreEqual(fullName, states.Single().FullName); + CollectionAssert.Contains(BrowserTestBase.Log, "signs in as owner"); + } + + [TestMethod] + public async Task ClassInitialize_RunsWithATestContext() + { + SignInFixture.ClassContext = null; + + await RunAsync(MakeTest(nameof(SignInFixture.UsesContext))); + + Assert.IsNotNull(SignInFixture.ClassContext, "the visual runner should run [ClassInitialize]"); + Assert.AreEqual(typeof(SignInFixture).FullName, SignInFixture.ClassContext.FullyQualifiedTestClassName); + } + + [TestMethod] + public void Discovery_ExpandsDataRows() + { + var discovery = new TestDiscovery(NullLogger.Instance); + + var tests = discovery.Discover([typeof(RunnerDataRowFixture).Assembly.Location], nameof(RunnerDataRowFixture)); + + CollectionAssert.AreEquivalent( + new[] + { + $"{typeof(RunnerDataRowFixture).FullName}.Doubles (1,2)", + $"{typeof(RunnerDataRowFixture).FullName}.Doubles (2,4)", + }, + tests.Select(t => t.FullName).ToArray()); + Assert.IsTrue(tests.All(t => t.Arguments is { Length: 2 })); + } +} + +/// A data-driven test MSTest runs as well, discovered by the visual runner in the test above. +[TestClass] +public class RunnerDataRowFixture +{ + [DataTestMethod] + [DataRow(1, 2)] + [DataRow(2, 4)] + public void Doubles(int value, int doubled) => Assert.AreEqual(doubled, value * 2); +}