public void Should_IgnoreMutants_WithoutCoveringTestsInfo_When_Tests_Have_Changed()
{
// Arrange
var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose);
var options = new StrykerOptions()
{
WithBaseline = false,
ProjectVersion = "version"
};
diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult
{
ChangedSourceFiles = new List <string>(),
ChangedTestFiles = new List <string> {
"C:/testfile.cs"
}
});
diffProvider.SetupGet(x => x.Tests).Returns(new TestSet());
var target = new SinceMutantFilter(diffProvider.Object);
var mutants = new List <Mutant>
{
new Mutant {
CoveringTests = TestsGuidList.NoTest()
}
};
// Act
var results = target.FilterMutants(mutants, new CsharpFileLeaf(), options);
// Assert
results.ShouldBeEmpty();
}
public TestRunResult(bool success, string message = null)
{
FailingTests = !success?TestsGuidList.EveryTest() : TestsGuidList.NoTest();
RanTests = TestsGuidList.EveryTest();
ResultMessage = message;
Duration = TimeSpan.Zero;
}
public static IProjectComponent CreateProjectWith(bool duplicateMutant = false, int mutationScore = 60)
{
var tree = CSharpSyntaxTree.ParseText("void M(){ int i = 0 + 8; }");
var originalNode = tree.GetRoot().DescendantNodes().OfType <BinaryExpressionSyntax>().First();
var mutation = new Mutation()
{
OriginalNode = originalNode,
ReplacementNode = SyntaxFactory.BinaryExpression(SyntaxKind.SubtractExpression, originalNode.Left, originalNode.Right),
DisplayName = "This name should display",
Type = Mutator.Arithmetic
};
var folder = new CsharpFolderComposite {
FullPath = "/home/user/src/project/", RelativePath = ""
};
int mutantCount = 0;
for (var i = 1; i <= 2; i++)
{
var addedFolder = new CsharpFolderComposite
{
RelativePath = $"{i}",
FullPath = $"/home/user/src/project/{i}",
};
folder.Add(addedFolder);
for (var y = 0; y <= 4; y++)
{
var m = new Collection <Mutant>();
addedFolder.Add(new CsharpFileLeaf()
{
RelativePath = $"{i}/SomeFile{y}.cs",
FullPath = $"/home/user/src/project/{i}/SomeFile{y}.cs",
Mutants = m,
SourceCode = "void M(){ int i = 0 + 8; }"
});
for (var z = 0; z <= 5; z++)
{
m.Add(new Mutant()
{
Id = duplicateMutant ? 2 : ++mutantCount,
ResultStatus = 100 / 6 * z < mutationScore ? MutantStatus.Killed : MutantStatus.Survived,
Mutation = mutation,
CoveringTests = TestsGuidList.EveryTest()
});
}
}
}
return(folder);
}
public void FilterMutants_FiltersNoMutants_IfTestsChanged()
{
// Arrange
var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose);
var options = new StrykerOptions()
{
WithBaseline = false,
ProjectVersion = "version"
};
diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult
{
ChangedSourceFiles = new List <string>(),
ChangedTestFiles = new List <string> {
"C:/testfile1.cs"
}
});
var tests = new TestSet();
var test1 = new TestDescription(Guid.NewGuid(), "name1", "C:/testfile1.cs");
var test2 = new TestDescription(Guid.NewGuid(), "name2", "C:/testfile2.cs");
tests.RegisterTests(new[] { test1, test2 });
diffProvider.SetupGet(x => x.Tests).Returns(tests);
var target = new SinceMutantFilter(diffProvider.Object);
var testFile1 = new TestsGuidList(new [] { test1 });
var testFile2 = new TestsGuidList(new [] { test2 });
var expectedToStay1 = new Mutant {
CoveringTests = testFile1
};
var expectedToStay2 = new Mutant {
CoveringTests = testFile1
};
var newMutant = new Mutant {
CoveringTests = testFile2
};
var mutants = new List <Mutant>
{
expectedToStay1,
expectedToStay2,
newMutant
};
// Act
var results = target.FilterMutants(mutants, new CsharpFileLeaf(), options);
// Assert
results.ShouldBe(new [] { expectedToStay1, expectedToStay2 });
}
public void InitialTestProcess_ShouldThrowExceptionOnFail()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var failedTest = Guid.NewGuid();
var successfulTest = Guid.NewGuid();
var ranTests = new TestsGuidList(failedTest, successfulTest);
var failedTests = new TestsGuidList(failedTest);
testRunnerMock.Setup(x => x.InitialTest()).Returns(new TestRunResult(ranTests, failedTests, TestsGuidList.NoTest(), string.Empty, TimeSpan.Zero));
testRunnerMock.Setup(x => x.CaptureCoverage(It.IsAny <List <Mutant> >()))
.Returns(new TestRunResult(true));
testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet());
Assert.Throws <InputException>(() => _target.InitialTest(_options, testRunnerMock.Object));
}
public void MutationTestExecutor_FailedTestShouldBeKilled()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var mutant = new Mutant {
Id = 1, CoveringTests = TestsGuidList.EveryTest()
};
testRunnerMock.Setup(x => x.TestMultipleMutants(null, It.IsAny <IReadOnlyList <Mutant> >(), null)).Returns(new TestRunResult(false));
var target = new MutationTestExecutor(testRunnerMock.Object);
target.Test(new List <Mutant> {
mutant
}, null, null);
mutant.ResultStatus.ShouldBe(MutantStatus.Killed);
testRunnerMock.Verify(x => x.TestMultipleMutants(null, It.IsAny <IReadOnlyList <Mutant> >(), null), Times.Once);
}
private void TestMutants(IEnumerable <Mutant> mutantsToTest)
{
var mutantGroups = BuildMutantGroupsForTest(mutantsToTest.ToList());
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = _options.Concurrency
};
var testsFailingInitialy = Input.InitialTestRun.Result.FailingTests.GetGuids().ToHashSet();
Parallel.ForEach(mutantGroups, parallelOptions, mutants =>
{
var reportedMutants = new HashSet <Mutant>();
bool testUpdateHandler(IReadOnlyList <Mutant> testedMutants, ITestGuids failedTests, ITestGuids ranTests, ITestGuids timedOutTest)
{
var continueTestRun = _options.OptimizationMode.HasFlag(OptimizationModes.DisableBail);
if (testsFailingInitialy.Count > 0 && failedTests.GetGuids().Any(id => testsFailingInitialy.Contains(id)))
{
// some of the failing tests where failing without any mutation
// we discard those tests
failedTests = new TestsGuidList(
failedTests.GetGuids().Where(t => !testsFailingInitialy.Contains(t)));
}
foreach (var mutant in testedMutants)
{
mutant.AnalyzeTestRun(failedTests, ranTests, timedOutTest);
if (mutant.ResultStatus == MutantStatus.NotRun)
{
continueTestRun = true; // Not all mutants in this group were tested so we continue
}
OnMutantTested(mutant, reportedMutants); // Report on mutant that has been tested
}
return(continueTestRun);
}
_mutationTestExecutor.Test(mutants, Input.InitialTestRun.TimeoutValueCalculator, testUpdateHandler);
OnMutantsTested(mutants, reportedMutants);
});
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants)
{
_logger.LogDebug($"{RunnerId}: Capturing coverage.");
if (CantUseStrykerDataCollector())
{
_logger.LogDebug($"{RunnerId}: project does not support StrykerDataCollector. Coverage data is simulated. Upgrade test proj" +
$" to@ NetCore 2.0+");
// can't capture coverage info
foreach (var mutant in mutants)
{
mutant.CoveringTests = TestsGuidList.EveryTest();
}
}
else
{
var testResults = RunTestSession(null, true, GenerateRunSettings(null, false, true, null));
ParseResultsForCoverage(testResults.TestResults, mutants);
}
return(new TestRunResult(true));
}
public void InitialTestProcess_ShouldNotThrowIfAFewTestsFail()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var test1 = Guid.NewGuid();
var testList = new List <Guid>(10);
testList.Add(test1);
for (int i = testList.Count; i < testList.Capacity; i++)
{
testList.Add(Guid.NewGuid());
}
var ranTests = new TestsGuidList(testList);
var failedTests = new TestsGuidList(test1);
testRunnerMock.Setup(x => x.InitialTest()).Returns(new TestRunResult(ranTests, failedTests, TestsGuidList.NoTest(), string.Empty, TimeSpan.Zero));
testRunnerMock.Setup(x => x.CaptureCoverage(It.IsAny <List <Mutant> >()))
.Returns(new TestRunResult(true));
testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet());
_target.InitialTest(_options, testRunnerMock.Object);
}
public void MutationTestExecutor_TimeoutShouldBePassedToProcessTimeout()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var mutant = new Mutant {
Id = 1, CoveringTests = TestsGuidList.EveryTest()
};
testRunnerMock.Setup(x => x.TestMultipleMutants(It.IsAny <ITimeoutValueCalculator>(), It.IsAny <IReadOnlyList <Mutant> >(), null)).
Returns(TestRunResult.TimedOut(TestsGuidList.NoTest(), TestsGuidList.NoTest(), TestsGuidList.EveryTest(), "", TimeSpan.Zero));
var target = new MutationTestExecutor(testRunnerMock.Object);
var timeoutValueCalculator = new TimeoutValueCalculator(500);
target.Test(new List <Mutant> {
mutant
}, timeoutValueCalculator, null);
mutant.ResultStatus.ShouldBe(MutantStatus.Timeout);
testRunnerMock.Verify(x => x.TestMultipleMutants(timeoutValueCalculator, It.IsAny <IReadOnlyList <Mutant> >(), null), Times.Once);
}
private void SetCoveringTests(IReadOnlyCollection <Mutant> mutantsToScan)
{
// force static mutants to be tested against all tests.
if (!_options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest))
{
foreach (var mutant in mutantsToScan.Where(mutant => mutant.IsStaticValue))
{
mutant.CoveringTests = TestsGuidList.EveryTest();
}
}
foreach (var mutant in mutantsToScan)
{
if (mutant.CoveringTests.IsEmpty)
{
mutant.ResultStatus = MutantStatus.NoCoverage;
mutant.ResultStatusReason = "Mutant has no test coverage";
}
else if (!_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest))
{
mutant.CoveringTests = TestsGuidList.EveryTest();
}
}
}
private TestRunResult BuildTestRunResult(IRunResults testResults, int expectedTests, bool compressAll = true)
{
var resultAsArray = testResults.TestResults.ToArray();
var testCases = resultAsArray.Select(t => t.TestCase.Id).Distinct();
var ranTestsCount = testCases.Count();
var timeout = !_aborted && ranTestsCount < expectedTests;
var ranTests = (compressAll && ranTestsCount >= DiscoverTests().Count) ? (ITestGuids)TestsGuidList.EveryTest() : new WrappedGuidsEnumeration(testCases);
var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(t => t.TestCase.Id);
if (ranTests.IsEmpty && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0))
{
_logger.LogTrace($"{RunnerId}: Initial Test session reports 0 result and 0 stuck tests.");
}
var duration = testResults.TestResults.Aggregate(new TimeSpan(), (span, result) => span.Add(result.Duration));
var message = string.Join(Environment.NewLine,
resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage))
.Select(tr => $"{tr.DisplayName}{Environment.NewLine}{Environment.NewLine}{tr.ErrorMessage}"));
var failedTestsDescription = new WrappedGuidsEnumeration(failedTests);
var timedOutTests = new WrappedGuidsEnumeration(testResults.TestsInTimeout?.Select(t => t.Id));
return(timeout
? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message, duration)
: new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message, duration));
}
public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, ITestGuids>();
var needAll = true;
ICollection <Guid> testCases;
int?timeOutMs = timeoutCalc?.DefaultTimeout;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest))
{
needAll = false;
foreach (var mutant in mutants)
{
ITestListDescription tests;
if ((mutant.IsStaticValue && !_options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = TestsGuidList.EveryTest();
needAll = true;
}
else
{
tests = mutant.CoveringTests;
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetGuids()).ToList();
_logger.LogTrace($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestsGuidList.NoTest(), TestsGuidList.NoTest(), TestsGuidList.NoTest(), "Mutants are not covered by any test!", TimeSpan.Zero));
}
if (timeoutCalc != null && testCases != null)
{
// compute time out
var duration = (int)testCases.Select(id => _vsTests[id].InitialRunTime.TotalMilliseconds).Sum();
timeOutMs = timeoutCalc.CalculateTimeoutValue(duration);
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants[0].Id, TestsGuidList.EveryTest());
testCases = null;
}
}
else
{
testCases = null;
}
var numberTestCases = testCases == null ? 0 : testCases.Count;
var expectedTests = needAll ? DiscoverTests().Count : numberTestCases;
void HandleUpdate(IRunResults handler)
{
var handlerTestResults = handler.TestResults;
if (mutants == null)
{
return;
}
var tests = handlerTestResults.Count == DiscoverTests().Count
? (ITestGuids)TestsGuidList.EveryTest()
: new WrappedGuidsEnumeration(handlerTestResults.Select(t => t.TestCase.Id));
var failedTest = new WrappedGuidsEnumeration(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(t => t.TestCase.Id));
var timedOutTest = new WrappedGuidsEnumeration(handler.TestsInTimeout?.Select(t => t.Id));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, timedOutTest);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
_logger.LogDebug("Using {0} ms as testrun timeout", timeOutMs.ToString() ?? "no");
var testResults = RunTestSession(mutantTestsMap, needAll, GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
return(BuildTestRunResult(testResults, expectedTests));
}
