From d01dc7130719e68717a4bde9946b1222230d5c65 Mon Sep 17 00:00:00 2001 From: Torrey Betts Date: Fri, 23 Oct 2026 15:45:21 -0400 Subject: [PATCH] Add Motus.Testing.MSTest.v4, the MSTest integration built for MSTest 4 Motus.Testing.MSTest is built against MSTest 3, and MSTest 4 renamed the assemblies its types live in. MSTest recognizes an inherited [TestInitialize] and [TestCleanup] by the exact identity of its attribute types, so a project on MSTest 4 deriving from MotusTestBase had its setup and cleanup ignored (MSTEST0082), failed to compile as soon as it touched TestContext (CS0012), and could not be discovered at all. MotusTestMethodAttribute also overrode Execute, which MSTest 4 replaced with ExecuteAsync. NuGet chooses a package's assets by target framework, never by the version of another dependency, so one package cannot carry a build for each MSTest major. The new Motus.Testing.MSTest.v4 package compiles the same source files, linked rather than copied, against MSTest.TestFramework 4.0.0, with the same namespace and types, so moving a suite to MSTest 4 is a package reference change. The only source that differs is MotusTestMethodAttribute, which under MSTEST_V4 overrides ExecuteAsync and passes the caller's file and line to MSTest, taking them from the wrapped attribute when [MotusTestClass] wraps one. The name follows Microsoft.Playwright.MSTest.v4. Motus.Testing.MSTest is unchanged for MSTest 3 users, but now declares MSTest.TestFramework [3.5.2, 4.0.0), so a project on MSTest 4 that references it gets NU1608 at restore instead of tests that silently never get a page. A new test project on MSTest 4.5.1, set up as dotnet new mstest sets it up, runs the issue's sample (TestContext.WriteLine in a MotusTestBase subclass) against a browser, runs the retry through MSTest 4 itself, and treats MSTEST0082 as an error. The release workflow packs the new package, and the docs and package READMEs say which package goes with which MSTest. Fixes #19 Signed-off-by: Torrey Betts --- .github/workflows/motus-release.yml | 1 + Motus.sln | 30 ++++ README.md | 4 +- docs/architecture/overview.md | 3 +- docs/getting-started.md | 3 +- docs/guides/testing-frameworks.md | 13 +- docs/migration/from-playwright.md | 4 +- docs/migration/from-selenium.md | 2 +- .../Motus.Testing.MSTest.v4.csproj | 44 +++++ .../Motus.Testing.MSTest.csproj | 7 +- .../MotusTestMethodAttribute.cs | 54 ++++++ src/Motus.Testing.MSTest/README.md | 10 ++ src/Motus.Testing/README.md | 2 +- src/Motus/README.md | 3 +- .../LifecycleAttributeTests.cs | 31 ++++ .../LostBrowserRetryTests.cs | 29 +++ .../MSTestSettings.cs | 1 + .../Motus.Testing.MSTest.v4.Tests.csproj | 29 +++ .../MotusTestMethodAttributeTests.cs | 169 ++++++++++++++++++ .../SmokeTests.cs | 45 +++++ 20 files changed, 473 insertions(+), 11 deletions(-) create mode 100644 src/Motus.Testing.MSTest.v4/Motus.Testing.MSTest.v4.csproj create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/LifecycleAttributeTests.cs create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/LostBrowserRetryTests.cs create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/MSTestSettings.cs create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/Motus.Testing.MSTest.v4.Tests.csproj create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/MotusTestMethodAttributeTests.cs create mode 100644 tests/Motus.Testing.MSTest.v4.Tests/SmokeTests.cs diff --git a/.github/workflows/motus-release.yml b/.github/workflows/motus-release.yml index bc0ad62..051362d 100644 --- a/.github/workflows/motus-release.yml +++ b/.github/workflows/motus-release.yml @@ -53,6 +53,7 @@ jobs: dotnet pack src/Motus.Recorder/Motus.Recorder.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages dotnet pack src/Motus.Testing/Motus.Testing.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages dotnet pack src/Motus.Testing.MSTest/Motus.Testing.MSTest.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages + dotnet pack src/Motus.Testing.MSTest.v4/Motus.Testing.MSTest.v4.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages dotnet pack src/Motus.Testing.xUnit/Motus.Testing.xUnit.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages dotnet pack src/Motus.Testing.NUnit/Motus.Testing.NUnit.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages dotnet pack src/Motus.Cli/Motus.Cli.csproj -c Release -p:Version=${{ steps.version.outputs.version }} -o packages diff --git a/Motus.sln b/Motus.sln index 3a61c76..c3fbc62 100644 --- a/Motus.sln +++ b/Motus.sln @@ -53,6 +53,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Mcp.Http", "src\Motus EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.xUnit.Tests", "tests\Motus.Testing.xUnit.Tests\Motus.Testing.xUnit.Tests.csproj", "{C65BD8A2-DF4C-49C2-A070-8998E55D1C38}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4", "src\Motus.Testing.MSTest.v4\Motus.Testing.MSTest.v4.csproj", "{14C675DB-0985-4110-8DBE-585D5AD24571}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Motus.Testing.MSTest.v4.Tests", "tests\Motus.Testing.MSTest.v4.Tests\Motus.Testing.MSTest.v4.Tests.csproj", "{88F3ADB4-C36D-4DDD-8976-0B4645EE0381}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -327,6 +331,30 @@ Global {C65BD8A2-DF4C-49C2-A070-8998E55D1C38}.Release|x64.Build.0 = Release|Any CPU {C65BD8A2-DF4C-49C2-A070-8998E55D1C38}.Release|x86.ActiveCfg = Release|Any CPU {C65BD8A2-DF4C-49C2-A070-8998E55D1C38}.Release|x86.Build.0 = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|x64.ActiveCfg = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|x64.Build.0 = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|x86.ActiveCfg = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Debug|x86.Build.0 = Debug|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|Any CPU.ActiveCfg = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|Any CPU.Build.0 = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|x64.ActiveCfg = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|x64.Build.0 = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|x86.ActiveCfg = Release|Any CPU + {14C675DB-0985-4110-8DBE-585D5AD24571}.Release|x86.Build.0 = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|Any CPU.Build.0 = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|x64.ActiveCfg = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|x64.Build.0 = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|x86.ActiveCfg = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Debug|x86.Build.0 = Debug|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|Any CPU.ActiveCfg = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|Any CPU.Build.0 = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x64.ActiveCfg = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x64.Build.0 = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.ActiveCfg = Release|Any CPU + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -354,6 +382,8 @@ Global {110037A6-2F59-4273-B115-5047BDC39CE6} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} {F2B42529-6278-4AC5-B1E4-49E55A95FDEA} = {C2BE1865-1543-4EBB-9E95-92EE083BF967} {C65BD8A2-DF4C-49C2-A070-8998E55D1C38} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} + {14C675DB-0985-4110-8DBE-585D5AD24571} = {C2BE1865-1543-4EBB-9E95-92EE083BF967} + {88F3ADB4-C36D-4DDD-8976-0B4645EE0381} = {47F5729D-17CB-4046-88FA-BB0F9F660B6E} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3A99AF87-BFA9-49CF-A20A-71753BE54A53} diff --git a/README.md b/README.md index e83388c..4b2f6d1 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The result is a framework that talks directly to Chromium and Firefox (CDP and W ```bash dotnet add package Motus -dotnet add package Motus.Testing.MSTest # or Motus.Testing.xUnit / Motus.Testing.NUnit +dotnet add package Motus.Testing.MSTest.v4 # MSTest 4; or Motus.Testing.MSTest (MSTest 3) / Motus.Testing.xUnit / Motus.Testing.NUnit dotnet tool install --global Motus.Cli ``` @@ -276,7 +276,7 @@ All CDP types are source-generated at build time from the protocol JSON schema. | `Motus.Recorder` | Action capture, selector inference, POM generation | | `Motus.Runner` | Blazor visual test runner with live screencast and timeline | | `Motus.Cli` | `motus` CLI tool | -| `Motus.Testing.*` | Test framework integrations (MSTest, xUnit, NUnit) | +| `Motus.Testing.*` | Test framework integrations (MSTest 3 and 4, xUnit, NUnit) | ## The Extension Model diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index e255f29..d5b1e55 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -14,7 +14,8 @@ Motus is a .NET browser automation framework that communicates directly with Chr | `Motus.Runner` | Blazor visual test runner with live screencast and action timeline | | `Motus.Cli` | `motus` CLI tool (`run`, `record`, `codegen`, `screenshot`, `pdf`, `trace`, `install`) | | `Motus.Testing` | Shared base types for test framework integrations | -| `Motus.Testing.MSTest` | MSTest integration (`MotusTestBase`, lifecycle management) | +| `Motus.Testing.MSTest` | MSTest 3 integration (`MotusTestBase`, lifecycle management) | +| `Motus.Testing.MSTest.v4` | The same source built for MSTest 4 | | `Motus.Testing.xUnit` | xUnit integration | | `Motus.Testing.NUnit` | NUnit integration | | `Motus.Samples` | Runnable usage examples | diff --git a/docs/getting-started.md b/docs/getting-started.md index ade362c..2f12810 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -16,7 +16,8 @@ Add the Motus core package and the integration for your test framework: dotnet add package Motus # Pick one test framework integration -dotnet add package Motus.Testing.MSTest +dotnet add package Motus.Testing.MSTest.v4 # MSTest 4, which `dotnet new mstest` uses +dotnet add package Motus.Testing.MSTest # MSTest 3 dotnet add package Motus.Testing.NUnit dotnet add package Motus.Testing.xUnit ``` diff --git a/docs/guides/testing-frameworks.md b/docs/guides/testing-frameworks.md index 66f6ec1..a7c23f4 100644 --- a/docs/guides/testing-frameworks.md +++ b/docs/guides/testing-frameworks.md @@ -6,7 +6,18 @@ Motus ships first-class test helpers for MSTest, NUnit, and xUnit. Each package ## MSTest -Package: `Motus.Testing.MSTest` +Package: `Motus.Testing.MSTest` for MSTest 3, `Motus.Testing.MSTest.v4` for MSTest 4 + +### MSTest 3 and MSTest 4 + +MSTest 4 renamed the assemblies its types live in, and MSTest recognizes an inherited `[TestInitialize]` or `[TestCleanup]` only by the exact identity of its own attribute types. A base class built against one major is invisible to a test project on the other: its setup and cleanup never run, and a test that touches `TestContext` does not compile. Motus therefore ships the MSTest integration twice, built from the same source: + +| Your test project uses | Reference | +|---|---| +| MSTest 4.x, which `dotnet new mstest` creates | `Motus.Testing.MSTest.v4` | +| MSTest 3.x | `Motus.Testing.MSTest` | + +Both packages use the namespace `Motus.Testing.MSTest` and contain the same types, so moving a suite from MSTest 3 to 4 changes the package reference and nothing in the tests. Reference one or the other, not both. `Motus.Testing.MSTest` accepts MSTest 3 only, so a project on MSTest 4 that references it gets a `NU1608` warning at restore rather than tests that quietly never get a page. ### MotusTestBase diff --git a/docs/migration/from-playwright.md b/docs/migration/from-playwright.md index 4a8ce9e..952a200 100644 --- a/docs/migration/from-playwright.md +++ b/docs/migration/from-playwright.md @@ -386,13 +386,15 @@ Environment variables prefixed with `MOTUS_` override individual config values a ### 1. Replace the NuGet package -Remove `Microsoft.Playwright` (and `Microsoft.Playwright.MSTest` if present). Add the Motus packages: +Remove `Microsoft.Playwright` (and `Microsoft.Playwright.MSTest` or `Microsoft.Playwright.MSTest.v4` if present). Add the Motus packages: ```xml ``` +On MSTest 4, reference `Motus.Testing.MSTest.v4` in place of `Motus.Testing.MSTest`, the way `Microsoft.Playwright.MSTest.v4` stands in for `Microsoft.Playwright.MSTest`. The namespace is `Motus.Testing.MSTest` in both. + ### 2. Remove the Playwright CLI install step Delete any `playwright install` or `pwsh bin/Debug/.../playwright.ps1 install` commands from your build scripts. Motus discovers the system browser or a browser installed via the Motus browser installer; there is no generated script to run. diff --git a/docs/migration/from-selenium.md b/docs/migration/from-selenium.md index 1714d7d..4b1ba88 100644 --- a/docs/migration/from-selenium.md +++ b/docs/migration/from-selenium.md @@ -246,7 +246,7 @@ var page = await context.NewPageAsync(); **1. Remove WebDriver NuGet packages.** -Remove `Selenium.WebDriver`, `Selenium.WebDriver.ChromeDriver`, and any related packages. Add `Motus` and, if using MSTest, `Motus.Testing.MSTest`. +Remove `Selenium.WebDriver`, `Selenium.WebDriver.ChromeDriver`, and any related packages. Add `Motus` and, if using MSTest, `Motus.Testing.MSTest` (MSTest 3) or `Motus.Testing.MSTest.v4` (MSTest 4). **2. Replace driver initialization.** diff --git a/src/Motus.Testing.MSTest.v4/Motus.Testing.MSTest.v4.csproj b/src/Motus.Testing.MSTest.v4/Motus.Testing.MSTest.v4.csproj new file mode 100644 index 0000000..c05e4d5 --- /dev/null +++ b/src/Motus.Testing.MSTest.v4/Motus.Testing.MSTest.v4.csproj @@ -0,0 +1,44 @@ + + + + net8.0;net10.0 + latest + enable + enable + True + $(NoWarn);1591 + true + true + Motus.Testing.MSTest.v4 + Motus.Testing.MSTest.v4 + MSTest 4 integration for Motus browser automation. Provides a base test class with per-test browser context isolation. For MSTest 3, use Motus.Testing.MSTest. + false + + Motus.Testing.MSTest + $(DefineConstants);MSTEST_V4 + + + + + + + + + + + + + + + + + + diff --git a/src/Motus.Testing.MSTest/Motus.Testing.MSTest.csproj b/src/Motus.Testing.MSTest/Motus.Testing.MSTest.csproj index 446e09e..08c54c9 100644 --- a/src/Motus.Testing.MSTest/Motus.Testing.MSTest.csproj +++ b/src/Motus.Testing.MSTest/Motus.Testing.MSTest.csproj @@ -11,7 +11,7 @@ true Motus.Testing.MSTest Motus.Testing.MSTest - MSTest integration for Motus browser automation. Provides a base test class with per-test browser context isolation. + MSTest 3 integration for Motus browser automation. Provides a base test class with per-test browser context isolation. For MSTest 4, use Motus.Testing.MSTest.v4. false @@ -28,8 +28,11 @@ + - + diff --git a/src/Motus.Testing.MSTest/MotusTestMethodAttribute.cs b/src/Motus.Testing.MSTest/MotusTestMethodAttribute.cs index 8de7448..21a0476 100644 --- a/src/Motus.Testing.MSTest/MotusTestMethodAttribute.cs +++ b/src/Motus.Testing.MSTest/MotusTestMethodAttribute.cs @@ -1,4 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +#if MSTEST_V4 +using System.Runtime.CompilerServices; +#endif // Motus has a test result of its own, for the runs its own runner reports. This file is about // MSTest's, so the one name it borrows from Motus is named on its own. @@ -27,6 +30,34 @@ public class MotusTestMethodAttribute : TestMethodAttribute { private readonly TestMethodAttribute? _inner; +#if MSTEST_V4 + /// + /// Applies to a test method directly, in place of [TestMethod]. + /// + /// + /// The caller's file and line are filled in by the compiler and passed on to MSTest, which uses + /// them to take the test explorer to the test's source. + /// + public MotusTestMethodAttribute([CallerFilePath] string callerFilePath = "", [CallerLineNumber] int callerLineNumber = -1) + : base(callerFilePath, callerLineNumber) + { + } + + /// + /// Wraps another kind of test method, so what that one does is kept and only the retry is added. + /// + /// + /// Called from code rather than written on a method, so the source location is the wrapped + /// attribute's, not this constructor's caller. + /// +#pragma warning disable MSTEST0057 // the location comes from the wrapped attribute + public MotusTestMethodAttribute(TestMethodAttribute? inner) + : base(inner?.DeclaringFilePath ?? "", inner?.DeclaringLineNumber ?? -1) +#pragma warning restore MSTEST0057 + { + _inner = inner; + } +#else /// /// Applies to a test method directly, in place of [TestMethod]. /// @@ -41,12 +72,34 @@ public MotusTestMethodAttribute(TestMethodAttribute? inner) { _inner = inner; } +#endif /// /// How many further attempts a lost browser is worth. Zero runs the test once, whatever happens. /// public int Retries { get; set; } = 1; + // MSTest 4 runs a test through ExecuteAsync, and MSTest 3 through Execute, which 4 removed. +#if MSTEST_V4 + /// + public override async Task ExecuteAsync(ITestMethod testMethod) + { + // No ConfigureAwait(false): each attempt resumes on the context MSTest ran the first one + // on, which an attribute being wrapped may depend on. + var results = await RunAsync(testMethod); + + for (int attempt = 1; attempt <= Retries && LostTheBrowser(results); attempt++) + { + AnnounceRetry(testMethod, attempt); + results = await RunAsync(testMethod); + } + + return results; + } + + private Task RunAsync(ITestMethod testMethod) + => _inner is not null ? _inner.ExecuteAsync(testMethod) : base.ExecuteAsync(testMethod); +#else /// public override TestResult[] Execute(ITestMethod testMethod) { @@ -63,6 +116,7 @@ public override TestResult[] Execute(ITestMethod testMethod) private TestResult[] Run(ITestMethod testMethod) => _inner is not null ? _inner.Execute(testMethod) : base.Execute(testMethod); +#endif private static bool LostTheBrowser(TestResult[]? results) => results is not null && Array.Exists(results, result => BrowserFailure.IsBrowserLost(result.TestFailureException)); diff --git a/src/Motus.Testing.MSTest/README.md b/src/Motus.Testing.MSTest/README.md index e41cef0..2bd3213 100644 --- a/src/Motus.Testing.MSTest/README.md +++ b/src/Motus.Testing.MSTest/README.md @@ -10,10 +10,20 @@ Provides `MotusTestBase`, a base class that shares a single browser across all t ## Installation +For MSTest 4, which `dotnet new mstest` creates: + +```shell +dotnet add package Motus.Testing.MSTest.v4 +``` + +For MSTest 3: + ```shell dotnet add package Motus.Testing.MSTest ``` +The two packages are built from the same source for each MSTest major, with the same namespace and types, so moving between them changes the package reference and nothing in the tests. A base class built against one MSTest major has its setup and cleanup ignored by the other, which is why there are two. + ## Quick Start ```csharp diff --git a/src/Motus.Testing/README.md b/src/Motus.Testing/README.md index 0528af6..1b9cb49 100644 --- a/src/Motus.Testing/README.md +++ b/src/Motus.Testing/README.md @@ -27,6 +27,6 @@ dotnet add package Motus.Testing This package depends on [Motus](https://www.nuget.org/packages/Motus) and [Motus.Abstractions](https://www.nuget.org/packages/Motus.Abstractions). Most consumers should install one of the framework-specific packages instead: -- [Motus.Testing.MSTest](https://www.nuget.org/packages/Motus.Testing.MSTest) +- [Motus.Testing.MSTest](https://www.nuget.org/packages/Motus.Testing.MSTest) (MSTest 3) or [Motus.Testing.MSTest.v4](https://www.nuget.org/packages/Motus.Testing.MSTest.v4) (MSTest 4) - [Motus.Testing.xUnit](https://www.nuget.org/packages/Motus.Testing.xUnit) - [Motus.Testing.NUnit](https://www.nuget.org/packages/Motus.Testing.NUnit) diff --git a/src/Motus/README.md b/src/Motus/README.md index 91db72d..a6fef98 100644 --- a/src/Motus/README.md +++ b/src/Motus/README.md @@ -44,7 +44,8 @@ await page.Locator("a").ClickAsync(); | [Motus.Analyzers](https://www.nuget.org/packages/Motus.Analyzers) | Roslyn analyzers for common automation mistakes | | [Motus.Recorder](https://www.nuget.org/packages/Motus.Recorder) | Record browser interactions and emit test code | | [Motus.Testing](https://www.nuget.org/packages/Motus.Testing) | Shared browser fixture for test frameworks | -| [Motus.Testing.MSTest](https://www.nuget.org/packages/Motus.Testing.MSTest) | MSTest integration | +| [Motus.Testing.MSTest](https://www.nuget.org/packages/Motus.Testing.MSTest) | MSTest 3 integration | +| [Motus.Testing.MSTest.v4](https://www.nuget.org/packages/Motus.Testing.MSTest.v4) | MSTest 4 integration | | [Motus.Testing.xUnit](https://www.nuget.org/packages/Motus.Testing.xUnit) | xUnit integration | | [Motus.Testing.NUnit](https://www.nuget.org/packages/Motus.Testing.NUnit) | NUnit integration | | [Motus.Cli](https://www.nuget.org/packages/Motus.Cli) | CLI for test execution, recording, and browser management | diff --git a/tests/Motus.Testing.MSTest.v4.Tests/LifecycleAttributeTests.cs b/tests/Motus.Testing.MSTest.v4.Tests/LifecycleAttributeTests.cs new file mode 100644 index 0000000..ab1f129 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/LifecycleAttributeTests.cs @@ -0,0 +1,31 @@ +using System.Reflection; +using Motus.Testing.MSTest; + +namespace Motus.Testing.MSTest.v4.Tests; + +/// +/// MSTest finds an inherited [TestInitialize] and [TestCleanup], and sets +/// TestContext, by the exact identity of its own types. These hold when the base class was +/// built against the MSTest this project runs on, and fail when it was built against another major. +/// +[TestClass] +public class LifecycleAttributeTests +{ + [TestMethod] + public void SetupAndCleanup_AreMarkedWithThisMSTestsAttributes() + { + var setup = typeof(MotusTestBase).GetMethod(nameof(MotusTestBase.MotusTestInitialize)); + var cleanup = typeof(MotusTestBase).GetMethod(nameof(MotusTestBase.MotusTestCleanup)); + + Assert.IsNotNull(setup?.GetCustomAttribute(), "MSTest would never run the setup"); + Assert.IsNotNull(cleanup?.GetCustomAttribute(), "MSTest would never run the cleanup"); + } + + [TestMethod] + public void TheTestContextProperty_IsThisMSTestsTestContext() + { + var property = typeof(MotusTestBase).GetProperty(nameof(MotusTestBase.TestContext)); + + Assert.AreEqual(typeof(TestContext), property?.PropertyType); + } +} diff --git a/tests/Motus.Testing.MSTest.v4.Tests/LostBrowserRetryTests.cs b/tests/Motus.Testing.MSTest.v4.Tests/LostBrowserRetryTests.cs new file mode 100644 index 0000000..482d9e8 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/LostBrowserRetryTests.cs @@ -0,0 +1,29 @@ +using Motus.Testing.MSTest; +using MotusTargetClosedException = Motus.Abstractions.MotusTargetClosedException; + +namespace Motus.Testing.MSTest.v4.Tests; + +/// +/// Runs the retry through MSTest 4 itself rather than through a stand-in test method. +/// +[TestClass] +public class LostBrowserRetryTests +{ + private static int s_attempts; + private static int s_setups; + + [TestInitialize] + public void Setup() => s_setups++; + + [MotusTestMethod] + public void ATestWhoseBrowserWentAway_RunsAgainWithASetupOfItsOwn() + { + s_attempts++; + + if (s_attempts == 1) + throw new MotusTargetClosedException("page", "A1", "CDP WebSocket disconnected."); + + Assert.AreEqual(2, s_attempts, "the test should have been run a second time"); + Assert.AreEqual(2, s_setups, "the second attempt should have had its own setup"); + } +} diff --git a/tests/Motus.Testing.MSTest.v4.Tests/MSTestSettings.cs b/tests/Motus.Testing.MSTest.v4.Tests/MSTestSettings.cs new file mode 100644 index 0000000..300f5b1 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/MSTestSettings.cs @@ -0,0 +1 @@ +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] diff --git a/tests/Motus.Testing.MSTest.v4.Tests/Motus.Testing.MSTest.v4.Tests.csproj b/tests/Motus.Testing.MSTest.v4.Tests/Motus.Testing.MSTest.v4.Tests.csproj new file mode 100644 index 0000000..36e2f14 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/Motus.Testing.MSTest.v4.Tests.csproj @@ -0,0 +1,29 @@ + + + + net8.0;net10.0 + latest + enable + enable + false + true + + $(WarningsAsErrors);MSTEST0082 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/tests/Motus.Testing.MSTest.v4.Tests/MotusTestMethodAttributeTests.cs b/tests/Motus.Testing.MSTest.v4.Tests/MotusTestMethodAttributeTests.cs new file mode 100644 index 0000000..ae8aa70 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/MotusTestMethodAttributeTests.cs @@ -0,0 +1,169 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using Motus.Testing.MSTest; +using MotusTargetClosedException = Motus.Abstractions.MotusTargetClosedException; + +namespace Motus.Testing.MSTest.v4.Tests; + +/// +/// The retry through MSTest 4's asynchronous ExecuteAsync, which replaced the +/// Execute the MSTest 3 build overrides. +/// +[TestClass] +public class MotusTestMethodAttributeTests +{ + [TestMethod] + public async Task ARunThatLostTheBrowser_IsTried_Again() + { + var testMethod = new ScriptedTestMethod(LostBrowser(), Passed()); + + var results = await new MotusTestMethodAttribute().ExecuteAsync(testMethod); + + Assert.AreEqual(2, testMethod.Invocations); + Assert.AreEqual(UnitTestOutcome.Passed, results[0].Outcome); + } + + [TestMethod] + public async Task AnAssertionThatFailed_IsLeftAlone() + { + var testMethod = new ScriptedTestMethod( + Failed(new AssertFailedException("expected 'motus', found ''")), Passed()); + + var results = await new MotusTestMethodAttribute().ExecuteAsync(testMethod); + + Assert.AreEqual(1, testMethod.Invocations); + Assert.AreEqual(UnitTestOutcome.Failed, results[0].Outcome); + } + + [TestMethod] + public async Task ABrowserThatKeepsDying_IsGivenUpOn() + { + var testMethod = new ScriptedTestMethod(LostBrowser(), LostBrowser(), LostBrowser(), Passed()); + + var results = await new MotusTestMethodAttribute { Retries = 2 }.ExecuteAsync(testMethod); + + Assert.AreEqual(3, testMethod.Invocations); + Assert.AreEqual(UnitTestOutcome.Failed, results[0].Outcome); + } + + [TestMethod] + public async Task RetriesOfZero_RunsTheTestOnce() + { + var testMethod = new ScriptedTestMethod(LostBrowser(), Passed()); + + await new MotusTestMethodAttribute { Retries = 0 }.ExecuteAsync(testMethod); + + Assert.AreEqual(1, testMethod.Invocations); + } + + [TestMethod] + public void TheClassAttribute_GivesEveryTestTheRetry() + { + var wrapped = new MotusTestClassAttribute { Retries = 3 } + .GetTestMethodAttribute(new TestMethodAttribute()); + + var motus = wrapped as MotusTestMethodAttribute; + Assert.IsNotNull(motus, "a plain test method should come back wrapped"); + Assert.AreEqual(3, motus.Retries); + } + + [TestMethod] + public void TheClassAttribute_LeavesAMethodThatAskedForItself() + { + var asked = new MotusTestMethodAttribute { Retries = 5 }; + + var wrapped = new MotusTestClassAttribute { Retries = 1 }.GetTestMethodAttribute(asked); + + Assert.AreSame(asked, wrapped); + } + + [TestMethod] + public async Task TheClassAttribute_KeepsWhatAnotherKindOfTestMethodDoes() + { + var inner = new CountingTestMethodAttribute(); + var wrapped = new MotusTestClassAttribute().GetTestMethodAttribute(inner); + var testMethod = new ScriptedTestMethod(Passed()); + + await wrapped!.ExecuteAsync(testMethod); + + Assert.AreEqual(1, inner.Executions, "the wrapped attribute should be the one that runs the test"); + } + + [TestMethod] + public void AWrappedTestMethod_KeepsItsSourceLocation() + { + // MSTest 4 takes the test explorer to a test's source by the location its attribute + // carries. A wrapper that dropped it would leave every test in the class without one. + var inner = MethodBase.GetCurrentMethod()!.GetCustomAttribute()!; + + var wrapped = new MotusTestClassAttribute().GetTestMethodAttribute(inner)!; + + StringAssert.EndsWith(inner.DeclaringFilePath, "MotusTestMethodAttributeTests.cs"); + Assert.AreEqual(inner.DeclaringFilePath, wrapped.DeclaringFilePath); + Assert.AreEqual(inner.DeclaringLineNumber, wrapped.DeclaringLineNumber); + } + + [TestMethod] + public void AppliedToAMethod_ItRecordsWhereItWasWritten() + { + var attribute = typeof(LostBrowserRetryTests) + .GetMethod(nameof(LostBrowserRetryTests.ATestWhoseBrowserWentAway_RunsAgainWithASetupOfItsOwn))! + .GetCustomAttribute()!; + + StringAssert.EndsWith(attribute.DeclaringFilePath, "LostBrowserRetryTests.cs"); + Assert.IsTrue(attribute.DeclaringLineNumber > 0); + } + + private static TestResult Passed() => new() { Outcome = UnitTestOutcome.Passed }; + + private static TestResult Failed(Exception failure) + => new() { Outcome = UnitTestOutcome.Failed, TestFailureException = failure }; + + private static TestResult LostBrowser() + => Failed(new MotusTargetClosedException("page", "A1", "CDP WebSocket disconnected.")); + + /// + /// Hands back a fixed run of outcomes, one per attempt, and counts the attempts. + /// + private sealed class ScriptedTestMethod(params TestResult[] results) : ITestMethod + { + internal int Invocations { get; private set; } + + public Task InvokeAsync(object?[]? arguments) + { + var result = results[Math.Min(Invocations, results.Length - 1)]; + Invocations++; + return Task.FromResult(result); + } + + public string TestMethodName => nameof(ScriptedTestMethod); + public string TestClassName => "Motus.Testing.MSTest.v4.Tests"; + public Type ReturnType => typeof(Task); + public object?[]? Arguments => null; + public ParameterInfo[] ParameterTypes => []; + public MethodInfo MethodInfo => typeof(ScriptedTestMethod).GetMethod(nameof(InvokeAsync))!; + + public Attribute[] GetAllAttributes() => []; + + public TAttributeType[] GetAttributes() + where TAttributeType : Attribute => []; + } + + /// + /// Stands in for a test method attribute that does something of its own, such as running a row + /// of data per attempt. + /// + private sealed class CountingTestMethodAttribute( + [CallerFilePath] string callerFilePath = "", + [CallerLineNumber] int callerLineNumber = -1) + : TestMethodAttribute(callerFilePath, callerLineNumber) + { + internal int Executions { get; private set; } + + public override Task ExecuteAsync(ITestMethod testMethod) + { + Executions++; + return base.ExecuteAsync(testMethod); + } + } +} diff --git a/tests/Motus.Testing.MSTest.v4.Tests/SmokeTests.cs b/tests/Motus.Testing.MSTest.v4.Tests/SmokeTests.cs new file mode 100644 index 0000000..57814f8 --- /dev/null +++ b/tests/Motus.Testing.MSTest.v4.Tests/SmokeTests.cs @@ -0,0 +1,45 @@ +using Motus.Testing.MSTest; + +namespace Motus.Testing.MSTest.v4.Tests; + +/// +/// The sample that MSTest 4 support was asked for with: a class deriving from +/// that writes to its . Built +/// against the MSTest 3 package it does not compile, and without the TestContext line it compiles +/// and is then never discovered. +/// +[TestClass] +[TestCategory("Integration")] +public class SmokeTests : MotusTestBase +{ + [TestMethod] + public async Task Opens() + { + TestContext.WriteLine("starting"); + await Page.GotoAsync("data:text/html,

hi

"); + + Assert.AreEqual("hi", await Page.Locator("p").TextContentAsync()); + } + + [TestMethod] + public void TheTestContext_IsTheOneMSTestSetForThisTest() + => Assert.AreEqual(nameof(TheTestContext_IsTheOneMSTestSetForThisTest), TestContext.TestName); +} + +/// +/// The same through , which hands every test to +/// and so runs it through that attribute's MSTest 4 path. +/// +[MotusTestClass] +[TestCategory("Integration")] +public class MotusTestClassSmokeTests : MotusTestBase +{ + [TestMethod] + public async Task Opens() + { + TestContext.WriteLine("starting"); + await Page.GotoAsync("data:text/html,

hi

"); + + Assert.AreEqual("hi", await Page.Locator("p").TextContentAsync()); + } +}