/// <summary> /// Executes the mutation test session for the given test project. /// Algorithm: /// 1. Build project /// 2. Calculate for each test which mutations they cover. /// 3. Rebuild to remove injected code. /// 4. Run test session. /// 4a. Generate optimized test runs. /// 5. Build report. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <TestProjectReportModel> Test(MutationSessionProgressTracker progressTracker, CancellationToken cancellationToken = default) { // Build project progressTracker.LogBeginPreBuilding(); var projectInfo = await BuildProject(progressTracker, _testProjectPath); progressTracker.LogEndPreBuilding(); // Copy project N times progressTracker.LogBeginProjectDuplication(_parallel); var testProjectCopier = new TestProjectDuplicator(Directory.GetParent(projectInfo.AssemblyPath).FullName); var duplications = testProjectCopier.MakeInitialCopies(projectInfo, _parallel); // Begin code coverage on first project. var duplicationPool = new TestProjectDuplicationPool(duplications); var coverageProject = duplicationPool.TakeTestProject(); var coverageProjectInfo = GetTestProjectInfo(coverageProject, projectInfo); // Measure the test coverage progressTracker.LogBeginCoverage(); PrepareAssembliesForCodeCoverage(coverageProjectInfo); var coverageTimer = new Stopwatch(); coverageTimer.Start(); var coverage = await RunCoverage(coverageProject.TestProjectFile.FullFilePath(), cancellationToken); coverageTimer.Stop(); var timeout = _createTimeOut(coverageTimer); GC.Collect(); GC.WaitForPendingFinalizers(); coverageProject.MarkAsFree(); // Start test session. var testsPerMutation = GroupMutationsWithTests(coverage); return(StartMutationTestSession(coverageProjectInfo, testsPerMutation, progressTracker, timeout, duplicationPool)); }
/// <summary> /// Starts the mutation test session and returns the report with results. /// </summary> /// <param name="testProjectInfo"></param> /// <param name="testsPerMutation"></param> /// <param name="sessionProgressTracker"></param> /// <param name="coverageTestRunTime"></param> /// <param name="testProjectDuplicationPool"></param> /// <returns></returns> private TestProjectReportModel StartMutationTestSession(TestProjectInfo testProjectInfo, Dictionary <RegisteredCoverage, HashSet <string> > testsPerMutation, MutationSessionProgressTracker sessionProgressTracker, TimeSpan coverageTestRunTime, TestProjectDuplicationPool testProjectDuplicationPool) { // Generate the mutation test runs for the mutation session. var defaultMutationTestRunGenerator = new DefaultMutationTestRunGenerator(); var runs = defaultMutationTestRunGenerator.GenerateMutationTestRuns(testsPerMutation, testProjectInfo, _mutationLevel); // Double the time the code coverage took such that test runs have some time run their tests (needs to be in seconds). var maxTestDuration = TimeSpan.FromSeconds((coverageTestRunTime * 2).Seconds); var reportBuilder = new TestProjectReportModelBuilder(testProjectInfo.TestModule.Name); var allRunsStopwatch = new Stopwatch(); allRunsStopwatch.Start(); var mutationTestRuns = runs.ToList(); var totalRunsCount = mutationTestRuns.Count(); var mutationCount = mutationTestRuns.Sum(x => x.MutationCount); var completedRuns = 0; var failedRuns = 0; sessionProgressTracker.LogBeginTestSession(totalRunsCount, mutationCount, maxTestDuration); // Stores timed out mutations which will be excluded from test runs if they occur. // Timed out mutations will be removed because they can cause serious test delays. var timedOutMutations = new List <MutationVariantIdentifier>(); async Task RunTestRun(IMutationTestRun testRun) { var testProject = testProjectDuplicationPool.AcquireTestProject(); try { testRun.InitializeMutations(testProject, timedOutMutations); var singRunsStopwatch = new Stopwatch(); singRunsStopwatch.Start(); var results = await testRun.RunMutationTestAsync(maxTestDuration, sessionProgressTracker, _testHostRunFactory, testProject, _testHostLogger); if (results != null) { foreach (var testResult in results) { // Store the timed out mutations such that they can be excluded. timedOutMutations.AddRange(testResult.GetTimedOutTests()); // For each mutation add it to the report builder. reportBuilder.AddTestResult(testResult.TestResults, testResult.Mutations, singRunsStopwatch.Elapsed); } singRunsStopwatch.Stop(); singRunsStopwatch.Reset(); } } catch (Exception e) { _testHostLogger.LogError(e, "The test process encountered an unexpected error."); sessionProgressTracker.Log( $"The test process encountered an unexpected error. Continuing without this test run. Please consider to submit an github issue. {e}", LogMessageType.Error); failedRuns += 1; } finally { lock (this) { completedRuns += 1; sessionProgressTracker.LogTestRunUpdate(completedRuns, totalRunsCount, failedRuns); } testProject.MarkAsFree(); } } var tasks = from testRun in mutationTestRuns select RunTestRun(testRun); Task.WaitAll(tasks.ToArray()); allRunsStopwatch.Stop(); var report = reportBuilder.Build(allRunsStopwatch.Elapsed, totalRunsCount); sessionProgressTracker.LogEndTestSession(allRunsStopwatch.Elapsed, completedRuns, mutationCount, report.ScorePercentage); return(report); }