public TestRunResult(bool success, string message = null)
{
FailingTests = !success?TestListDescription.EveryTest() : TestListDescription.NoTest();
RanTests = TestListDescription.EveryTest();
ResultMessage = message;
}
public void MutationTestExecutor_TimeoutShouldBePassedToProcessTimeout()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var mutant = new Mutant {
Id = 1, MustRunAgainstAllTests = true
};
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), mutant, null)).
Returns(TestRunResult.TimedOut(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.EveryTest(), ""));
var target = new MutationTestExecutor(testRunnerMock.Object);
target.Test(new List <Mutant> {
mutant
}, 1999, null);
mutant.ResultStatus.ShouldBe(MutantStatus.Timeout);
testRunnerMock.Verify(x => x.RunAll(1999, mutant, null), Times.Once);
}
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, IList <string> >();
ICollection <TestCase> testCases = null;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var needAll = false;
foreach (var mutant in mutants)
{
List <string> tests;
if ((mutant.IsStaticValue && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = null;
needAll = true;
}
else
{
tests = mutant.CoveringTests.GetList().Select(t => t.Guid).ToList();
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetList()).Distinct().Select(t => _discoveredTests.First(tc => tc.Id.ToString() == t.Guid)).ToList();
_logger.LogDebug($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases.Select(x => x.FullyQualifiedName)))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.NoTest(), "Mutants are not covered by any test!"));
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants.FirstOrDefault().Id, new List <string>());
}
}
var expectedTests = testCases?.Count ?? DiscoverNumberOfTests();
void HandleUpdate(IRunResults handler)
{
if (mutants == null)
{
return;
}
var handlerTestResults = handler.TestResults;
var tests = handlerTestResults.Count == DiscoverNumberOfTests()
? TestListDescription.EveryTest()
: new TestListDescription(handlerTestResults.Select(tr => (TestDescription)tr.TestCase));
var failedTest = new TestListDescription(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(tr => (TestDescription)tr.TestCase));
var testsInProgress = new TestListDescription(handler.TestsInTimeout?.Select(t => (TestDescription)t));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, testsInProgress);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
var testResults = RunTestSession(testCases, GenerateRunSettings(timeoutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
var resultAsArray = testResults.TestResults.ToArray();
var timeout = (!_aborted && resultAsArray.Length < expectedTests);
var ranTests = resultAsArray.Length == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(resultAsArray.Select(tr => (TestDescription)tr.TestCase));
var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(tr => (TestDescription)tr.TestCase).ToImmutableArray();
if (ranTests.Count == 0 && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0))
{
_logger.LogDebug($"{RunnerId}: Test session reports 0 result and 0 stuck tests.");
}
var message = string.Join(Environment.NewLine,
resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage))
.Select(tr => tr.ErrorMessage));
var failedTestsDescription = new TestListDescription(failedTests);
var timedOutTests = new TestListDescription(testResults.TestsInTimeout?.Select(t => (TestDescription)t));
return(timeout ? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message) : new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message));
}