diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/AssemblyTestServerTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/AssemblyTestServerTests.cs index ed73b13c2..67822b09b 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/AssemblyTestServerTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/AssemblyTestServerTests.cs @@ -316,7 +316,7 @@ public async Task RunTestsAsync_ShouldCallClientRunTests() var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); listener.Complete(); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); @@ -324,7 +324,7 @@ public async Task RunTestsAsync_ShouldCallClientRunTests() var result = await server.RunTestsAsync(null); result.ShouldNotBeNull(); - _clientMock.Verify(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null), Times.Once); + _clientMock.Verify(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny()), Times.Once); } [TestMethod] @@ -336,14 +336,14 @@ public async Task RunTestsAsync_ShouldPassTestNodesToClient() var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); listener.Complete(); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), testNodes)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), testNodes, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); await server.StartAsync(); await server.RunTestsAsync(testNodes); - _clientMock.Verify(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), testNodes), Times.Once); + _clientMock.Verify(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), testNodes, It.IsAny()), Times.Once); } [TestMethod] @@ -359,8 +359,8 @@ public async Task RunTestsAsync_ShouldCollectTestResults() new TestNodeUpdate(failedNode, "parent") }; - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) - .Returns, TestNode[]?>(async (id, callback, _) => + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) + .Returns, TestNode[]?, CancellationToken>(async (id, callback, _, _) => { await callback(updates); var listener = new TestNodeUpdatesResponseListener(id, _ => Task.CompletedTask); @@ -383,7 +383,7 @@ public async Task RunTestsAsync_WithTimeout_ShouldReturnTimedOutFalse_WhenComple var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); listener.Complete(); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); @@ -401,7 +401,7 @@ public async Task RunTestsAsync_WithTimeout_ShouldReturnTimedOutTrue_WhenTimesOu // Listener that never completes var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); @@ -418,7 +418,7 @@ public async Task RunTestsAsync_WithTimeout_ShouldThrowTestHostCrashed_WhenProce // Listener that never completes (a crashed host never sends a completion signal) var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); @@ -439,7 +439,7 @@ public async Task RunTestsAsync_WithoutTimeout_ShouldThrowTestHostCrashed_WhenPr SetupSuccessfulConnection(); var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .ReturnsAsync(listener); using var server = CreateServer(); @@ -452,13 +452,68 @@ public async Task RunTestsAsync_WithoutTimeout_ShouldThrowTestHostCrashed_WhenPr async () => await server.RunTestsAsync(null)); } + [TestMethod] + public async Task RunTestsAsync_ShouldInvokeBailCallback_AndReturnCollectedResults() + { + SetupSuccessfulConnection(); + + var failedNode = new TestNode("uid-1", "Test1", "action", "failed"); + var updates = new[] { new TestNodeUpdate(failedNode, "parent") }; + + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) + .Returns, TestNode[]?, CancellationToken>(async (id, callback, _, _) => + { + await callback(updates); + var listener = new TestNodeUpdatesResponseListener(id, _ => Task.CompletedTask); + listener.Complete(); + return listener; + }); + + IReadOnlyList? observed = null; + using var server = CreateServer(); + await server.StartAsync(); + + var (results, timedOut) = await server.RunTestsAsync(null, TimeSpan.FromSeconds(10), batch => + { + observed = batch; + return true; // request bail on first batch + }); + + // The streamed results are surfaced to the bail callback and still returned to the caller. + observed.ShouldNotBeNull(); + observed!.Single().Node.Uid.ShouldBe("uid-1"); + results.Count.ShouldBe(1); + timedOut.ShouldBeFalse(); + } + + [TestMethod] + public async Task RunTestsAsync_ShouldPassCancellableToken_ToClient_ForBail() + { + SetupSuccessfulConnection(); + + CancellationToken capturedToken = default; + var listener = new TestNodeUpdatesResponseListener(Guid.NewGuid(), _ => Task.CompletedTask); + listener.Complete(); + + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) + .Callback, TestNode[]?, CancellationToken>((_, _, _, token) => capturedToken = token) + .ReturnsAsync(listener); + + using var server = CreateServer(); + await server.StartAsync(); + await server.RunTestsAsync(null, TimeSpan.FromSeconds(10), _ => false); + + // A real, cancellable token is handed to the client so a bail can propagate $/cancelRequest. + capturedToken.CanBeCanceled.ShouldBeTrue(); + } + [TestMethod] public async Task RunTestsAsync_WithTimeout_ShouldReturnTimedOutTrue_WhenRpcCallBlocks() { SetupSuccessfulConnection(); // RPC call that never returns (simulates server stuck in infinite loop) - _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null)) + _clientMock.Setup(c => c.RunTestsAsync(It.IsAny(), It.IsAny>(), null, It.IsAny())) .Returns(new TaskCompletionSource().Task); using var server = CreateServer(); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerTests.cs index f4942d242..6f48dcf3c 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerTests.cs @@ -691,6 +691,133 @@ bool Update(IReadOnlyList _, ITestIdentifiers failed, ITestIdentifiers result.ResultMessage.ShouldContain("crash"); // failure reason is surfaced, not swallowed } + // --- Bail-on-first-failure tests --- + // + // The MTP runner now mirrors the VsTest runner: streamed results are fed to the mutation update + // handler as they arrive, and once every tested mutant's fate is decided the remaining tests/assemblies + // are skipped. The update handler returns true while mutants are pending and always returns true when + // --disable-bail is set, so honouring its return value is all that is needed to honour that option. + + [TestMethod, Timeout(1000)] + public async Task TestMultipleMutantsAsync_BailsOut_SkippingRemainingAssemblies_OnceMutantIsKilled() + { + const string assemblyA = "/a.dll"; + const string assemblyB = "/b.dll"; + _testsByAssembly[assemblyA] = [new TestNode("a-uid", "TestA", "test", TestNodeStates.Failed)]; + _testsByAssembly[assemblyB] = [new TestNode("b-uid", "TestB", "test", TestNodeStates.Passed)]; + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns([assemblyA, assemblyB]); + + var mutant = new Mock(); + mutant.Setup(m => m.Id).Returns(1); + + var runAssemblies = new List(); + using var runner = new StreamingAssemblyRunner( + _testsByAssembly, _testDescriptions, _testSet, _discoveryLock, runAssemblies); + + // Bail (return false) as soon as any test has failed; continue otherwise. + bool Update(IReadOnlyList _, ITestIdentifiers failed, ITestIdentifiers __, ITestIdentifiers ___) + => !failed.GetIdentifiers().Any(); + + await runner.TestMultipleMutantsAsync(project.Object, null, [mutant.Object], Update); + + // Assembly A streamed a failing test, so the run bailed and never touched assembly B. + runAssemblies.ShouldBe([assemblyA]); + } + + [TestMethod, Timeout(1000)] + public async Task TestMultipleMutantsAsync_DoesNotBail_WhenUpdateHandlerKeepsRunning() + { + // Mirrors --disable-bail: the handler always returns true, so every assembly must run even + // after a test fails. + const string assemblyA = "/a.dll"; + const string assemblyB = "/b.dll"; + _testsByAssembly[assemblyA] = [new TestNode("a-uid", "TestA", "test", TestNodeStates.Failed)]; + _testsByAssembly[assemblyB] = [new TestNode("b-uid", "TestB", "test", TestNodeStates.Passed)]; + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns([assemblyA, assemblyB]); + + var mutant = new Mock(); + mutant.Setup(m => m.Id).Returns(1); + + var runAssemblies = new List(); + using var runner = new StreamingAssemblyRunner( + _testsByAssembly, _testDescriptions, _testSet, _discoveryLock, runAssemblies); + + bool Update(IReadOnlyList _, ITestIdentifiers __, ITestIdentifiers ___, ITestIdentifiers ____) => true; + + await runner.TestMultipleMutantsAsync(project.Object, null, [mutant.Object], Update); + + runAssemblies.ShouldBe([assemblyA, assemblyB]); + } + + [TestMethod, Timeout(1000)] + public async Task TestMultipleMutantsAsync_FeedsStreamedResultsToUpdateHandler_BeforeRunCompletes() + { + const string assembly = "/a.dll"; + _testsByAssembly[assembly] = [new TestNode("a-uid", "TestA", "test", TestNodeStates.Failed)]; + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns([assembly]); + + var mutant = new Mock(); + mutant.Setup(m => m.Id).Returns(1); + + var runAssemblies = new List(); + using var runner = new StreamingAssemblyRunner( + _testsByAssembly, _testDescriptions, _testSet, _discoveryLock, runAssemblies); + + ITestIdentifiers? capturedFailed = null; + bool Update(IReadOnlyList _, ITestIdentifiers failed, ITestIdentifiers __, ITestIdentifiers ___) + { + capturedFailed = failed; + return false; + } + + await runner.TestMultipleMutantsAsync(project.Object, null, [mutant.Object], Update); + + // The failing test was surfaced to the handler via the incremental streaming path. + capturedFailed.ShouldNotBeNull(); + capturedFailed!.GetIdentifiers().ShouldContain("a-uid"); + } + + /// + /// Simulates the server streaming each discovered test's result to the bail callback during an assembly + /// run, without starting a real test host. Records the assemblies that were actually run so a test can + /// assert which ones were skipped after a bail. + /// + private sealed class StreamingAssemblyRunner : SingleMicrosoftTestPlatformRunner + { + private readonly List _runAssemblies; + + public StreamingAssemblyRunner( + Dictionary> testsByAssembly, + Dictionary testDescriptions, + TestSet testSet, + object discoveryLock, + List runAssemblies) + : base(0, testsByAssembly, testDescriptions, testSet, discoveryLock, NullLogger.Instance) + => _runAssemblies = runAssemblies; + + internal override Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> RunAssemblyTestsAsync( + string assembly, ITimeoutValueCalculator? timeoutCalc, Func, bool>? shouldBail = null) + { + _runAssemblies.Add(assembly); + var discovered = GetDiscoveredTests(assembly) ?? []; + + // Simulate the server streaming the discovered tests' results to the bail callback. + shouldBail?.Invoke(discovered.Select(t => new TestNodeUpdate(t, "root")).ToList()); + + var result = BuildTestRunResult( + discovered.Select(t => new TestNodeUpdate(t, "root")).ToList(), + discovered.Count, + TimeSpan.Zero); + return Task.FromResult<(TestRunResult?, bool, List?)>((result, false, discovered)); + } + } + /// /// Simulates an assembly whose test host crashes: returns the /// failure sentinel produced by the real exception path, without starting any server process. @@ -709,7 +836,7 @@ public CrashingAssemblyRunner( => _discovered = discovered; internal override Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> RunAssemblyTestsAsync( - string assembly, ITimeoutValueCalculator? timeoutCalc) + string assembly, ITimeoutValueCalculator? timeoutCalc, Func, bool>? shouldBail = null) => Task.FromResult<(TestRunResult?, bool, List?)>( (new TestRunResult(false, "simulated test host crash"), false, _discovered)); } @@ -1448,7 +1575,7 @@ public TimeoutSimulatingRunner( : base(id, testsByAssembly, testDescriptions, testSet, discoveryLock, logger) { } internal override Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> RunAssemblyTestsAsync( - string assembly, ITimeoutValueCalculator? timeoutCalc) + string assembly, ITimeoutValueCalculator? timeoutCalc, Func, bool>? shouldBail = null) { var discoveredTests = GetDiscoveredTests(assembly); var result = new TestRunResult( @@ -1475,7 +1602,7 @@ public NoTimeoutSimulatingRunner( : base(id, testsByAssembly, testDescriptions, testSet, discoveryLock, logger) { } internal override Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> RunAssemblyTestsAsync( - string assembly, ITimeoutValueCalculator? timeoutCalc) + string assembly, ITimeoutValueCalculator? timeoutCalc, Func, bool>? shouldBail = null) { var discoveredTests = GetDiscoveredTests(assembly); var result = new TestRunResult( diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/AssemblyTestServer.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/AssemblyTestServer.cs index 967ffa9b4..36c5f9e1c 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/AssemblyTestServer.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/AssemblyTestServer.cs @@ -42,6 +42,8 @@ public AssemblyTestServer( public bool IsInitialized => _isInitialized; + public Dictionary EnvironmentVariables => _environmentVariables; + /// /// True when the server has been initialized and its underlying process is still running. /// A test host that crashed mid-run (e.g. a mutation causing a fatal fault such as a @@ -131,7 +133,15 @@ public async Task> RunTestsAsync(TestNode[]? testsToRun) return results; } - public async Task<(List Results, bool TimedOut)> RunTestsAsync(TestNode[]? testsToRun, TimeSpan? timeout) + /// + /// Runs the given tests. When is supplied it is invoked with every batch of + /// streamed test results; returning true cancels the remaining tests (bail), leaving the test host + /// idle and reusable. The results gathered before the bail are still returned. + /// + public async Task<(List Results, bool TimedOut)> RunTestsAsync( + TestNode[]? testsToRun, + TimeSpan? timeout, + Func, bool>? shouldBail = null) { if (!_isInitialized || _client is null) { @@ -140,6 +150,7 @@ public async Task> RunTestsAsync(TestNode[]? testsToRun) var runId = Guid.NewGuid(); var testResults = new System.Collections.Concurrent.ConcurrentBag(); + using var bailSource = new CancellationTokenSource(); Func onUpdate = updates => { @@ -147,6 +158,13 @@ public async Task> RunTestsAsync(TestNode[]? testsToRun) { testResults.Add(update); } + + if (shouldBail is not null && !bailSource.IsCancellationRequested && shouldBail(updates)) + { + _logger.LogDebug("{RunnerId}: Each tested mutant's fate is decided; bailing out of the remaining tests for {Assembly}", _runnerId, _assembly); + bailSource.Cancel(); + } + return Task.CompletedTask; }; @@ -156,9 +174,13 @@ public async Task> RunTestsAsync(TestNode[]? testsToRun) try { // The RPC call itself can block when the server is stuck (e.g. infinite loop in mutated code) - executeTestsResponse = await _client.RunTestsAsync(runId, onUpdate, testsToRun) + executeTestsResponse = await _client.RunTestsAsync(runId, onUpdate, testsToRun, bailSource.Token) .WaitAsync(timeout.Value).ConfigureAwait(false); } + catch (OperationCanceledException) when (bailSource.IsCancellationRequested) + { + return (testResults.ToList(), false); + } catch (TimeoutException ex) { _logger.LogDebug(ex, "{RunnerId}: Test run RPC call timed out for {Assembly}", _runnerId, _assembly); @@ -173,12 +195,20 @@ public async Task> RunTestsAsync(TestNode[]? testsToRun) return (testResults.ToList(), !completed); } - var response = await _client.RunTestsAsync(runId, onUpdate, testsToRun).ConfigureAwait(false); - var responseCompletion = response.WaitCompletionAsync(); - await Task.WhenAny(responseCompletion, _process!.WaitForExitAsync()).ConfigureAwait(false); - ThrowIfHostCrashed(responseCompletion); + try + { + var response = await _client.RunTestsAsync(runId, onUpdate, testsToRun, bailSource.Token).ConfigureAwait(false); + var responseCompletion = response.WaitCompletionAsync(); + await Task.WhenAny(responseCompletion, _process!.WaitForExitAsync()).ConfigureAwait(false); + ThrowIfHostCrashed(responseCompletion); + + await responseCompletion.ConfigureAwait(false); + } + catch (OperationCanceledException) when (bailSource.IsCancellationRequested) + { + return (testResults.ToList(), false); + } - await responseCompletion.ConfigureAwait(false); return (testResults.ToList(), false); } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/ITestingPlatformClient.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/ITestingPlatformClient.cs index 521669281..05938cc7f 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/ITestingPlatformClient.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/ITestingPlatformClient.cs @@ -11,5 +11,5 @@ public interface ITestingPlatformClient : IDisposable Task ExitAsync(bool gracefully = true); Task WaitServerProcessExitAsync(); Task DiscoverTestsAsync(Guid requestId, Func action, bool @checked = true); - Task RunTestsAsync(Guid requestId, Func action, TestNode[]? testNodes = null); + Task RunTestsAsync(Guid requestId, Func action, TestNode[]? testNodes = null, CancellationToken cancellationToken = default); } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 39d9f2d30..ccdafab84 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -139,7 +139,8 @@ private void WriteMutantIdToFile(int mutantId) { var envVars = new Dictionary { - ["STRYKER_MUTANT_FILE"] = _mutantFilePath + ["STRYKER_MUTANT_FILE"] = _mutantFilePath, + ["MinVerSkip"] = "true" }; ExternalEnvironmentVariables.Add(envVars); @@ -273,7 +274,9 @@ private async Task GetOrCreateServerAsync(string assembly) await deadServer.StopAsync(force: true).ConfigureAwait(false); } - var environmentVariables = BuildEnvironmentVariables(); + var environmentVariables = deadServer is not null + ? deadServer.EnvironmentVariables + : BuildEnvironmentVariables(); var server = new AssemblyTestServer(assembly, environmentVariables, _logger, RunnerId, _options); var started = await server.StartAsync().ConfigureAwait(false); @@ -397,6 +400,72 @@ internal async Task HandleAssemblyTimeoutAsync(string assembly, List d } } + /// + /// Accumulates streamed test results during a single mutation run and feeds them to the mutation + /// so a verdict can be reached before every test has run. Mirrors the + /// VsTest runner's incremental update handling: the handler returns false once every tested mutant + /// is resolved (and bail is not disabled), which is the signal to cancel the remaining tests. + /// + private sealed class BailState(IReadOnlyList? mutants, TestUpdateHandler? update) + { + private readonly IReadOnlyList? _mutants = mutants; + private readonly TestUpdateHandler? _update = update; + private readonly object _lock = new(); + private readonly HashSet _executed = []; + private readonly HashSet _failed = []; + private readonly HashSet _timedOut = []; + + public bool IsEnabled => _update is not null && _mutants is not null; + + public bool BailRequested { get; private set; } + + /// + /// Called with each batch of streamed test results. Returns true when the current run should be + /// cancelled because every tested mutant's fate is decided. + /// + public bool OnResultsUpdated(IReadOnlyList updates) + { + if (!IsEnabled) + { + return false; + } + + lock (_lock) + { + foreach (var node in updates.Select(u => u.Node)) + { + var state = node.ExecutionState; + if (!TestNodeStates.IsFinished(state)) + { + continue; + } + + _executed.Add(node.Uid); + if (TestNodeStates.IsFailure(state)) + { + _failed.Add(node.Uid); + } + else if (TestNodeStates.IsTimeout(state)) + { + _timedOut.Add(node.Uid); + } + } + + var failed = new TestIdentifierList(_failed); + var executed = new TestIdentifierList(_executed); + var timedOut = _timedOut.Count == 0 ? TestIdentifierList.NoTest() : new TestIdentifierList(_timedOut); + + var continueRun = _update!.Invoke(_mutants!, failed, executed, timedOut); + if (!continueRun) + { + BailRequested = true; + } + + return BailRequested; + } + } + } + private sealed class TestRunAccumulator { private readonly List _executedTests = []; @@ -474,6 +543,12 @@ internal async Task RunAllTestsAsync( TestUpdateHandler? update, ITimeoutValueCalculator? timeoutCalc = null) { + // Evaluate streamed results as they arrive so the run can bail out as soon as every tested mutant's + // fate is decided. The handler returns true while mutants remain pending and always returns true when + // --disable-bail is set, so respecting its return value here is all that is needed to honour that option. + var bailState = new BailState(mutants, update); + Func, bool>? shouldBail = bailState.IsEnabled ? bailState.OnResultsUpdated : null; + try { WriteMutantIdToFile(mutantId); @@ -482,7 +557,7 @@ internal async Task RunAllTestsAsync( foreach (var assembly in assemblies) { - var (result, timedOut, discoveredTests) = await RunAssemblyTestsAsync(assembly, timeoutCalc).ConfigureAwait(false); + var (result, timedOut, discoveredTests) = await RunAssemblyTestsAsync(assembly, timeoutCalc, shouldBail).ConfigureAwait(false); if (discoveredTests is not null) { @@ -499,6 +574,14 @@ internal async Task RunAllTestsAsync( { accumulator.Aggregate(result, discoveredTests); } + + if (bailState.BailRequested) + { + // Every tested mutant has been killed/resolved, so the remaining assemblies cannot change + // any verdict. Skip them just like the VsTest runner cancels its session on first failure. + _logger.LogDebug("{RunnerId}: Bailing out; skipping the remaining test assemblies", RunnerId); + break; + } } var executedTests = accumulator.BuildExecutedTests(); @@ -562,7 +645,8 @@ internal async Task RunAllTestsAsync( internal virtual async Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> RunAssemblyTestsAsync( string assembly, - ITimeoutValueCalculator? timeoutCalc) + ITimeoutValueCalculator? timeoutCalc, + Func, bool>? shouldBail = null) { if (!File.Exists(assembly)) { @@ -570,22 +654,23 @@ internal async Task RunAllTestsAsync( } var discoveredTests = GetDiscoveredTests(assembly); - + TimeSpan? timeout = null; if (timeoutCalc is not null && discoveredTests is not null) { timeout = CalculateAssemblyTimeout(discoveredTests, timeoutCalc, assembly); } - var (testResults, timedOut) = await RunAssemblyTestsInternalAsync(assembly, null, timeout).ConfigureAwait(false); - + var (testResults, timedOut) = await RunAssemblyTestsInternalAsync(assembly, null, timeout, shouldBail).ConfigureAwait(false); + return (testResults as TestRunResult, timedOut, discoveredTests); } internal async Task<(ITestRunResult Result, bool TimedOut)> RunAssemblyTestsInternalAsync( string assembly, Func? testUidFilter, - TimeSpan? timeout = null) + TimeSpan? timeout = null, + Func, bool>? shouldBail = null) { // A crashed test host tears down the RPC connection, so the run throws (rather than timing out). // Retry once on a freshly started server: a crash caused by a *previous* mutant then self-heals @@ -621,7 +706,7 @@ internal async Task RunAllTestsAsync( var testsToRun = tests?.Where(t => testUidFilter is null || testUidFilter(t)).ToArray(); - var (testResults, timedOut) = await server.RunTestsAsync(testsToRun, timeout).ConfigureAwait(false); + var (testResults, timedOut) = await server.RunTestsAsync(testsToRun, timeout, shouldBail).ConfigureAwait(false); var duration = DateTime.UtcNow - startTime; var result = BuildTestRunResult(testResults, tests?.Count ?? 0, duration); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/TestingPlatformClient.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/TestingPlatformClient.cs index 2ae435cf5..a9c2c3ae5 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/TestingPlatformClient.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/TestingPlatformClient.cs @@ -137,13 +137,17 @@ public async Task DiscoverTestsAsync(Guid requestId, Func RunTestsAsync(Guid requestId, Func action, TestNode[]? testNodes = null) + public async Task RunTestsAsync(Guid requestId, Func action, TestNode[]? testNodes = null, CancellationToken cancellationToken = default) => await CheckedInvokeAsync(async () => { - using CancellationTokenSource cancellationTokenSource = new(TimeSpan.FromMinutes(3)); + // The caller's token is linked with our hard upper bound so that cancelling the run (e.g. to bail + // out as soon as a mutant's fate is decided) propagates a $/cancelRequest to the test host, which + // stops scheduling the remaining tests instead of running them to completion. + using CancellationTokenSource timeoutSource = new(TimeSpan.FromMinutes(3)); + using var linkedSource = CancellationTokenSource.CreateLinkedTokenSource(timeoutSource.Token, cancellationToken); var runListener = new TestNodeUpdatesResponseListener(requestId, action); _targetHandler.RegisterResponseListener(runListener); - await JsonRpcClient.InvokeWithParameterObjectAsync("testing/runTests", new RunTestsRequest(RunId: requestId, TestCases: testNodes), cancellationToken: cancellationTokenSource.Token); + await JsonRpcClient.InvokeWithParameterObjectAsync("testing/runTests", new RunTestsRequest(RunId: requestId, TestCases: testNodes), cancellationToken: linkedSource.Token); return runListener; });