public TestRunResult(bool success, string message = null) { FailingTests = !success?TestListDescription.EveryTest() : TestListDescription.NoTest(); RanTests = TestListDescription.EveryTest(); ResultMessage = message; }
public void FilterMutants_FiltersNoMutants_IfTestsChanged() { // Arrange var baselineProvider = new Mock <IBaselineProvider>(); baselineProvider.Setup(x => x.Load(It.IsAny <string>()) ).Returns( Task.FromResult( JsonReport.Build(new StrykerOptions(), JsonReportTestHelper.CreateProjectWith().ToReadOnlyInputComponent()) )); var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose); var gitInfoProvider = new Mock <IGitInfoProvider>(); var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version"); diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult { ChangedSourceFiles = new List <string>(), ChangedTestFiles = new List <string> { "C:/testfile1.cs" } }); var target = new DiffMutantFilter(options, diffProvider.Object, baselineProvider.Object, gitInfoProvider.Object); var testFile1 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name1", "C:/testfile1.cs") }); var testFile2 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name2", "C:/testfile2.cs") }); var expectedToStay1 = new Mutant { CoveringTests = testFile1 }; var expectedToStay2 = new Mutant { CoveringTests = testFile1 }; var mutants = new List <Mutant> { expectedToStay1, expectedToStay2, new Mutant { CoveringTests = testFile2 } }; // Act var results = target.FilterMutants(mutants, new CsharpFileLeaf().ToReadOnly(), options); // Assert results.ShouldBe(new [] { expectedToStay1, expectedToStay2 }); }
public static IProjectComponent CreateProjectWith(bool duplicateMutant = false, int mutationScore = 60) { var tree = CSharpSyntaxTree.ParseText("void M(){ int i = 0 + 8; }"); var originalNode = tree.GetRoot().DescendantNodes().OfType <BinaryExpressionSyntax>().First(); var mutation = new Mutation() { OriginalNode = originalNode, ReplacementNode = SyntaxFactory.BinaryExpression(SyntaxKind.SubtractExpression, originalNode.Left, originalNode.Right), DisplayName = "This name should display", Type = Mutator.Arithmetic }; var folder = new CsharpFolderComposite { FullPath = "/home/user/src/project/", RelativePath = "" }; int mutantCount = 0; for (var i = 1; i <= 2; i++) { var addedFolder = new CsharpFolderComposite { RelativePath = $"{i}", FullPath = $"/home/user/src/project/{i}", }; folder.Add(addedFolder); for (var y = 0; y <= 4; y++) { var m = new Collection <Mutant>(); addedFolder.Add(new CsharpFileLeaf() { RelativePath = $"{i}/SomeFile{y}.cs", FullPath = $"/home/user/src/project/{i}/SomeFile{y}.cs", Mutants = m, SourceCode = "void M(){ int i = 0 + 8; }" }); for (var z = 0; z <= 5; z++) { m.Add(new Mutant() { Id = duplicateMutant ? 2 : ++mutantCount, ResultStatus = 100 / 6 * z < mutationScore ? MutantStatus.Killed : MutantStatus.Survived, Mutation = mutation, CoveringTests = TestListDescription.EveryTest() }); } } } return(folder); }
public void FilterMutants_FiltersNoMutants_IfTestsChanged() { // Arrange var baselineProvider = new Mock <IBaselineProvider>(); baselineProvider.Setup(x => x.Load(It.IsAny <string>()) ).Returns( Task.FromResult( JsonReport.Build(new StrykerOptions(), JsonReportTestHelper.CreateProjectWith()) )); var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose); var gitInfoProvider = new Mock <IGitInfoProvider>(); var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version"); diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult { ChangedFiles = new List <string>(), TestFilesChanged = new List <string> { "C:/testfile.cs" } }); var target = new DiffMutantFilter(options, diffProvider.Object, baselineProvider.Object, gitInfoProvider.Object); var testDescriptions = new List <TestDescription> { new TestDescription(Guid.NewGuid().ToString(), "name", "C:/testfile.cs") }; var testListDescription = new TestListDescription(testDescriptions); var mutants = new List <Mutant> { new Mutant { CoveringTests = testListDescription }, new Mutant(), new Mutant() }; // Act var results = target.FilterMutants(mutants, new FileLeaf(), options); // Assert results.Count().ShouldBe(1); }
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseUnloadAppDomain, bool cantUsePipe) { if (!_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) && !_flags.HasFlag(OptimizationFlags.CoverageBasedTest)) { return(new TestRunResult(true)); } if (cantUseUnloadAppDomain) { _logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. "); return(new TestRunResult(true)); } if (cantUsePipe) { _logger.LogWarning("Can't capture the test coverage as the target framework does not support pipes. "); return(new TestRunResult(true)); } var collector = new CoverageCollector(); collector.SetLogger(message => _logger.LogTrace(message)); var coverageEnvironment = new Dictionary <string, string>(); coverageEnvironment["Coverage"] = $"pipe:{_server.PipeName}"; var result = LaunchTestProcess(null, coverageEnvironment); if (!WaitOnLck(_lck, () => _lastMessage != null, 5000)) { // Failed to retrieve coverage data for testCase return(null); } var testedMutant = _lastMessage.Split(";")[0].Split(",").Select(int.Parse).ToList(); foreach (var mutant in mutants) { mutant.CoveringTests = testedMutant.Contains(mutant.Id) ? TestListDescription.EveryTest() : new TestListDescription(null); } return(result); }
public void FilterMutants_FiltersNoMutants_IfTestsChanged() { // Arrange var diffProvider = new Mock <IDiffProvider>(MockBehavior.Loose); var options = new StrykerOptions(compareToDashboard: false, projectVersion: "version"); diffProvider.Setup(x => x.ScanDiff()).Returns(new DiffResult { ChangedSourceFiles = new List <string>(), ChangedTestFiles = new List <string> { "C:/testfile1.cs" } }); var target = new DiffMutantFilter(diffProvider.Object); var testFile1 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name1", "C:/testfile1.cs") }); var testFile2 = new TestListDescription(new[] { new TestDescription(Guid.NewGuid().ToString(), "name2", "C:/testfile2.cs") }); var expectedToStay1 = new Mutant { CoveringTests = testFile1 }; var expectedToStay2 = new Mutant { CoveringTests = testFile1 }; var mutants = new List <Mutant> { expectedToStay1, expectedToStay2, new Mutant { CoveringTests = testFile2 } }; // Act var results = target.FilterMutants(mutants, new CsharpFileLeaf().ToReadOnly(), options); // Assert results.ShouldBe(new [] { expectedToStay1, expectedToStay2 }); }
public void MutationTestExecutor_TimeoutShouldBePassedToProcessTimeout() { var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict); var mutant = new Mutant { Id = 1, MustRunAgainstAllTests = true }; testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), mutant, null)). Returns(TestRunResult.TimedOut(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.EveryTest(), "")); var target = new MutationTestExecutor(testRunnerMock.Object); target.Test(new List <Mutant> { mutant }, 1999, null); mutant.ResultStatus.ShouldBe(MutantStatus.Timeout); testRunnerMock.Verify(x => x.RunAll(1999, mutant, null), Times.Once); }
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseAppDomain, bool cantUsePipe) { _logger.LogDebug($"{RunnerId}: Capturing coverage."); if (CantUseStrykerDataCollector()) { _logger.LogDebug($"{RunnerId}: project does not support StrykerDataCollector. Coverage data is simulated. Upgrade test proj" + $"" + $" to@ NetCore 2.0+"); // can't capture coverage info foreach (var mutant in mutants) { mutant.CoveringTests = TestListDescription.EveryTest(); } } else { var testResults = RunTestSession(null, GenerateRunSettings(null, false, true, null)); ParseResultsForCoverage(testResults.TestResults, mutants); } return(new TestRunResult(true)); }
public void GetCoverage() { var(targetFrameworkDoesNotSupportAppDomain, targetFrameworkDoesNotSupportPipe) = _input.ProjectInfo.ProjectUnderTestAnalyzerResult.CompatibilityModes; var mutantsToScan = _input.ProjectInfo.ProjectContents.Mutants.Where(x => x.ResultStatus == MutantStatus.NotRun).ToList(); foreach (var mutant in mutantsToScan) { mutant.CoveringTests = new TestListDescription(null); } var testResult = _mutationTestExecutor.TestRunner.CaptureCoverage(mutantsToScan, targetFrameworkDoesNotSupportPipe, targetFrameworkDoesNotSupportAppDomain); if (testResult.FailingTests.Count == 0) { // force static mutants to be tested against all tests. if (!_options.Optimizations.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) { foreach (var mutant in mutantsToScan.Where(mutant => mutant.IsStaticValue)) { mutant.MustRunAgainstAllTests = true; } } foreach (var mutant in mutantsToScan) { if (!mutant.MustRunAgainstAllTests && mutant.CoveringTests.IsEmpty) { mutant.ResultStatus = MutantStatus.NoCoverage; } else if (!_options.Optimizations.HasFlag(OptimizationFlags.CoverageBasedTest)) { mutant.CoveringTests = TestListDescription.EveryTest(); } } return; } _logger.LogWarning("Test run with no active mutation failed. Stryker failed to correctly generate the mutated assembly. Please report this issue on github with a logfile of this run."); throw new StrykerInputException("No active mutant testrun was not successful.", testResult.ResultMessage); }
private void SetCoveringTests(List <Mutant> mutantsToScan) { // force static mutants to be tested against all tests. if (!_options.Optimizations.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) { foreach (var mutant in mutantsToScan.Where(mutant => mutant.IsStaticValue)) { mutant.MustRunAgainstAllTests = true; } } foreach (var mutant in mutantsToScan) { if (!mutant.MustRunAgainstAllTests && mutant.CoveringTests.IsEmpty) { mutant.ResultStatus = MutantStatus.NoCoverage; mutant.ResultStatusReason = "Mutant has no test coverage"; } else if (!_options.Optimizations.HasFlag(OptimizationFlags.CoverageBasedTest)) { mutant.CoveringTests = TestListDescription.EveryTest(); } } }
public TestRunResult RunAll(int?timeoutMs, Mutant mutant, TestUpdateHandler update) { var envVars = mutant == null ? null : new Dictionary <string, string> { { "ActiveMutation", mutant.Id.ToString() } }; try { var result = LaunchTestProcess(timeoutMs, envVars); update?.Invoke(new[] { mutant }, result.RanTests, result.FailingTests, result.TimedOutTests); return(result); } catch (OperationCanceledException) { var emptyList = new TestListDescription(null); if (mutant != null) { mutant.ResultStatus = MutantStatus.Timeout; } return(TestRunResult.TimedOut(emptyList, emptyList, TestListDescription.EveryTest(), "time out")); } }
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update) { var mutantTestsMap = new Dictionary <int, IList <string> >(); ICollection <TestCase> testCases = null; if (mutants != null) { // if we optimize the number of tests to run if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest)) { var needAll = false; foreach (var mutant in mutants) { List <string> tests; if ((mutant.IsStaticValue && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests) { tests = null; needAll = true; } else { tests = mutant.CoveringTests.GetList().Select(t => t.Guid).ToList(); } mutantTestsMap.Add(mutant.Id, tests); } testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetList()).Distinct().Select(t => _discoveredTests.First(tc => tc.Id.ToString() == t.Guid)).ToList(); _logger.LogDebug($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " + $"against {(testCases == null ? "all tests." : string.Join(", ", testCases.Select(x => x.FullyQualifiedName)))}."); if (testCases?.Count == 0) { return(new TestRunResult(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.NoTest(), "Mutants are not covered by any test!")); } } else { if (mutants.Count > 1) { throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis."); } mutantTestsMap.Add(mutants.FirstOrDefault().Id, new List <string>()); } } var expectedTests = testCases?.Count ?? DiscoverNumberOfTests(); void HandleUpdate(IRunResults handler) { if (mutants == null) { return; } var handlerTestResults = handler.TestResults; var tests = handlerTestResults.Count == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(handlerTestResults.Select(tr => (TestDescription)tr.TestCase)); var failedTest = new TestListDescription(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed) .Select(tr => (TestDescription)tr.TestCase)); var testsInProgress = new TestListDescription(handler.TestsInTimeout?.Select(t => (TestDescription)t)); var remainingMutants = update?.Invoke(mutants, failedTest, tests, testsInProgress); if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted) { return; } // all mutants status have been resolved, we can stop _logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop."); _vsTestConsole.CancelTestRun(); _aborted = true; } var testResults = RunTestSession(testCases, GenerateRunSettings(timeoutMs, mutants != null, false, mutantTestsMap), HandleUpdate); var resultAsArray = testResults.TestResults.ToArray(); var timeout = (!_aborted && resultAsArray.Length < expectedTests); var ranTests = resultAsArray.Length == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(resultAsArray.Select(tr => (TestDescription)tr.TestCase)); var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(tr => (TestDescription)tr.TestCase).ToImmutableArray(); if (ranTests.Count == 0 && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0)) { _logger.LogDebug($"{RunnerId}: Test session reports 0 result and 0 stuck tests."); } var message = string.Join(Environment.NewLine, resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage)) .Select(tr => tr.ErrorMessage)); var failedTestsDescription = new TestListDescription(failedTests); var timedOutTests = new TestListDescription(testResults.TestsInTimeout?.Select(t => (TestDescription)t)); return(timeout ? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message) : new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message)); }