public TestRunResult(bool success, string message = null)
{
FailingTests = !success?TestListDescription.EveryTest() : TestListDescription.NoTest();
RanTests = TestListDescription.EveryTest();
ResultMessage = message;
}
public void FilterMutants_FiltersNoMutants_IfTestsChanged()
{
// Arrange
var baselineProvider = new Mock <IBaselineProvider>();
baselineProvider.Setup(x =>
x.Load(It.IsAny <string>())
).Returns(
Task.FromResult(
JsonReport.Build(new StrykerOptions(), JsonReportTestHelper.CreateProjectWith().ToReadOnlyInputComponent())
));
var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose);
var gitInfoProvider = new Mock <IGitInfoProvider>();
var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version");
diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult
{
ChangedSourceFiles = new List <string>(),
ChangedTestFiles = new List <string> {
"C:/testfile1.cs"
}
});
var target = new DiffMutantFilter(options, diffProvider.Object, baselineProvider.Object, gitInfoProvider.Object);
var testFile1 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name1", "C:/testfile1.cs") });
var testFile2 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name2", "C:/testfile2.cs") });
var expectedToStay1 = new Mutant
{
CoveringTests = testFile1
};
var expectedToStay2 = new Mutant
{
CoveringTests = testFile1
};
var mutants = new List <Mutant>
{
expectedToStay1,
expectedToStay2,
new Mutant
{
CoveringTests = testFile2
}
};
// Act
var results = target.FilterMutants(mutants, new CsharpFileLeaf().ToReadOnly(), options);
// Assert
results.ShouldBe(new [] { expectedToStay1, expectedToStay2 });
}
public static IProjectComponent CreateProjectWith(bool duplicateMutant = false, int mutationScore = 60)
{
var tree = CSharpSyntaxTree.ParseText("void M(){ int i = 0 + 8; }");
var originalNode = tree.GetRoot().DescendantNodes().OfType <BinaryExpressionSyntax>().First();
var mutation = new Mutation()
{
OriginalNode = originalNode,
ReplacementNode = SyntaxFactory.BinaryExpression(SyntaxKind.SubtractExpression, originalNode.Left, originalNode.Right),
DisplayName = "This name should display",
Type = Mutator.Arithmetic
};
var folder = new CsharpFolderComposite {
FullPath = "/home/user/src/project/", RelativePath = ""
};
int mutantCount = 0;
for (var i = 1; i <= 2; i++)
{
var addedFolder = new CsharpFolderComposite
{
RelativePath = $"{i}",
FullPath = $"/home/user/src/project/{i}",
};
folder.Add(addedFolder);
for (var y = 0; y <= 4; y++)
{
var m = new Collection <Mutant>();
addedFolder.Add(new CsharpFileLeaf()
{
RelativePath = $"{i}/SomeFile{y}.cs",
FullPath = $"/home/user/src/project/{i}/SomeFile{y}.cs",
Mutants = m,
SourceCode = "void M(){ int i = 0 + 8; }"
});
for (var z = 0; z <= 5; z++)
{
m.Add(new Mutant()
{
Id = duplicateMutant ? 2 : ++mutantCount,
ResultStatus = 100 / 6 * z < mutationScore ? MutantStatus.Killed : MutantStatus.Survived,
Mutation = mutation,
CoveringTests = TestListDescription.EveryTest()
});
}
}
}
return(folder);
}
public void FilterMutants_FiltersNoMutants_IfTestsChanged()
{
// Arrange
var baselineProvider = new Mock <IBaselineProvider>();
baselineProvider.Setup(x =>
x.Load(It.IsAny <string>())
).Returns(
Task.FromResult(
JsonReport.Build(new StrykerOptions(), JsonReportTestHelper.CreateProjectWith())
));
var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose);
var gitInfoProvider = new Mock <IGitInfoProvider>();
var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version");
diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult
{
ChangedFiles = new List <string>(),
TestFilesChanged = new List <string> {
"C:/testfile.cs"
}
});
var target = new DiffMutantFilter(options, diffProvider.Object, baselineProvider.Object, gitInfoProvider.Object);
var testDescriptions = new List <TestDescription> {
new TestDescription(Guid.NewGuid().ToString(), "name", "C:/testfile.cs")
};
var testListDescription = new TestListDescription(testDescriptions);
var mutants = new List <Mutant>
{
new Mutant {
CoveringTests = testListDescription
},
new Mutant(),
new Mutant()
};
// Act
var results = target.FilterMutants(mutants, new FileLeaf(), options);
// Assert
results.Count().ShouldBe(1);
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseUnloadAppDomain, bool cantUsePipe)
{
if (!_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) &&
!_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
return(new TestRunResult(true));
}
if (cantUseUnloadAppDomain)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. ");
return(new TestRunResult(true));
}
if (cantUsePipe)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support pipes. ");
return(new TestRunResult(true));
}
var collector = new CoverageCollector();
collector.SetLogger(message => _logger.LogTrace(message));
var coverageEnvironment = new Dictionary <string, string>();
coverageEnvironment["Coverage"] = $"pipe:{_server.PipeName}";
var result = LaunchTestProcess(null, coverageEnvironment);
if (!WaitOnLck(_lck, () => _lastMessage != null, 5000))
{
// Failed to retrieve coverage data for testCase
return(null);
}
var testedMutant = _lastMessage.Split(";")[0].Split(",").Select(int.Parse).ToList();
foreach (var mutant in mutants)
{
mutant.CoveringTests = testedMutant.Contains(mutant.Id) ? TestListDescription.EveryTest() : new TestListDescription(null);
}
return(result);
}
public void FilterMutants_FiltersNoMutants_IfTestsChanged()
{
// Arrange
var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose);
var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version");
diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult
{
ChangedSourceFiles = new List <string>(),
ChangedTestFiles = new List <string> {
"C:/testfile1.cs"
}
});
var target = new DiffMutantFilter(diffProvider.Object);
var testFile1 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name1", "C:/testfile1.cs") });
var testFile2 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name2", "C:/testfile2.cs") });
var expectedToStay1 = new Mutant
{
CoveringTests = testFile1
};
var expectedToStay2 = new Mutant
{
CoveringTests = testFile1
};
var mutants = new List <Mutant>
{
expectedToStay1,
expectedToStay2,
new Mutant
{
CoveringTests = testFile2
}
};
// Act
var results = target.FilterMutants(mutants, new CsharpFileLeaf().ToReadOnly(), options);
// Assert
results.ShouldBe(new [] { expectedToStay1, expectedToStay2 });
}
public void MutationTestExecutor_TimeoutShouldBePassedToProcessTimeout()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var mutant = new Mutant {
Id = 1, MustRunAgainstAllTests = true
};
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), mutant, null)).
Returns(TestRunResult.TimedOut(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.EveryTest(), ""));
var target = new MutationTestExecutor(testRunnerMock.Object);
target.Test(new List <Mutant> {
mutant
}, 1999, null);
mutant.ResultStatus.ShouldBe(MutantStatus.Timeout);
testRunnerMock.Verify(x => x.RunAll(1999, mutant, null), Times.Once);
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseAppDomain, bool cantUsePipe)
{
_logger.LogDebug($"{RunnerId}: Capturing coverage.");
if (CantUseStrykerDataCollector())
{
_logger.LogDebug($"{RunnerId}: project does not support StrykerDataCollector. Coverage data is simulated. Upgrade test proj" +
$"" +
$" to@ NetCore 2.0+");
// can't capture coverage info
foreach (var mutant in mutants)
{
mutant.CoveringTests = TestListDescription.EveryTest();
}
}
else
{
var testResults = RunTestSession(null, GenerateRunSettings(null, false, true, null));
ParseResultsForCoverage(testResults.TestResults, mutants);
}
return(new TestRunResult(true));
}
public void GetCoverage()
{
var(targetFrameworkDoesNotSupportAppDomain, targetFrameworkDoesNotSupportPipe) = _input.ProjectInfo.ProjectUnderTestAnalyzerResult.CompatibilityModes;
var mutantsToScan = _input.ProjectInfo.ProjectContents.Mutants.Where(x => x.ResultStatus == MutantStatus.NotRun).ToList();
foreach (var mutant in mutantsToScan)
{
mutant.CoveringTests = new TestListDescription(null);
}
var testResult = _mutationTestExecutor.TestRunner.CaptureCoverage(mutantsToScan, targetFrameworkDoesNotSupportPipe, targetFrameworkDoesNotSupportAppDomain);
if (testResult.FailingTests.Count == 0)
{
// force static mutants to be tested against all tests.
if (!_options.Optimizations.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
{
foreach (var mutant in mutantsToScan.Where(mutant => mutant.IsStaticValue))
{
mutant.MustRunAgainstAllTests = true;
}
}
foreach (var mutant in mutantsToScan)
{
if (!mutant.MustRunAgainstAllTests && mutant.CoveringTests.IsEmpty)
{
mutant.ResultStatus = MutantStatus.NoCoverage;
}
else if (!_options.Optimizations.HasFlag(OptimizationFlags.CoverageBasedTest))
{
mutant.CoveringTests = TestListDescription.EveryTest();
}
}
return;
}
_logger.LogWarning("Test run with no active mutation failed. Stryker failed to correctly generate the mutated assembly. Please report this issue on github with a logfile of this run.");
throw new StrykerInputException("No active mutant testrun was not successful.", testResult.ResultMessage);
}
private void SetCoveringTests(List <Mutant> mutantsToScan)
{
// force static mutants to be tested against all tests.
if (!_options.Optimizations.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
{
foreach (var mutant in mutantsToScan.Where(mutant => mutant.IsStaticValue))
{
mutant.MustRunAgainstAllTests = true;
}
}
foreach (var mutant in mutantsToScan)
{
if (!mutant.MustRunAgainstAllTests && mutant.CoveringTests.IsEmpty)
{
mutant.ResultStatus = MutantStatus.NoCoverage;
mutant.ResultStatusReason = "Mutant has no test coverage";
}
else if (!_options.Optimizations.HasFlag(OptimizationFlags.CoverageBasedTest))
{
mutant.CoveringTests = TestListDescription.EveryTest();
}
}
}
public TestRunResult RunAll(int?timeoutMs, Mutant mutant, TestUpdateHandler update)
{
var envVars = mutant == null ? null :
new Dictionary <string, string>
{
{ "ActiveMutation", mutant.Id.ToString() }
};
try
{
var result = LaunchTestProcess(timeoutMs, envVars);
update?.Invoke(new[] { mutant }, result.RanTests, result.FailingTests, result.TimedOutTests);
return(result);
}
catch (OperationCanceledException)
{
var emptyList = new TestListDescription(null);
if (mutant != null)
{
mutant.ResultStatus = MutantStatus.Timeout;
}
return(TestRunResult.TimedOut(emptyList, emptyList, TestListDescription.EveryTest(), "time out"));
}
}
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, IList <string> >();
ICollection <TestCase> testCases = null;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var needAll = false;
foreach (var mutant in mutants)
{
List <string> tests;
if ((mutant.IsStaticValue && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = null;
needAll = true;
}
else
{
tests = mutant.CoveringTests.GetList().Select(t => t.Guid).ToList();
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetList()).Distinct().Select(t => _discoveredTests.First(tc => tc.Id.ToString() == t.Guid)).ToList();
_logger.LogDebug($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases.Select(x => x.FullyQualifiedName)))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.NoTest(), "Mutants are not covered by any test!"));
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants.FirstOrDefault().Id, new List <string>());
}
}
var expectedTests = testCases?.Count ?? DiscoverNumberOfTests();
void HandleUpdate(IRunResults handler)
{
if (mutants == null)
{
return;
}
var handlerTestResults = handler.TestResults;
var tests = handlerTestResults.Count == DiscoverNumberOfTests()
? TestListDescription.EveryTest()
: new TestListDescription(handlerTestResults.Select(tr => (TestDescription)tr.TestCase));
var failedTest = new TestListDescription(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(tr => (TestDescription)tr.TestCase));
var testsInProgress = new TestListDescription(handler.TestsInTimeout?.Select(t => (TestDescription)t));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, testsInProgress);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
var testResults = RunTestSession(testCases, GenerateRunSettings(timeoutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
var resultAsArray = testResults.TestResults.ToArray();
var timeout = (!_aborted && resultAsArray.Length < expectedTests);
var ranTests = resultAsArray.Length == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(resultAsArray.Select(tr => (TestDescription)tr.TestCase));
var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(tr => (TestDescription)tr.TestCase).ToImmutableArray();
if (ranTests.Count == 0 && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0))
{
_logger.LogDebug($"{RunnerId}: Test session reports 0 result and 0 stuck tests.");
}
var message = string.Join(Environment.NewLine,
resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage))
.Select(tr => tr.ErrorMessage));
var failedTestsDescription = new TestListDescription(failedTests);
var timedOutTests = new TestListDescription(testResults.TestsInTimeout?.Select(t => (TestDescription)t));
return(timeout ? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message) : new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message));
}
