/// <summary> /// Injects the test project and all mutation assemblies with coverage injection code. /// This code that is injected will register calls to methods/tests. /// Those calls determine what tests cover which methods. /// </summary> /// <param name="projectInfo"></param> private void PrepareAssembliesForCodeCoverage(TestProjectInfo projectInfo) { TestCoverageInjector.Instance.InjectTestCoverage(projectInfo.TestModule); TestCoverageInjector.Instance.InjectModuleInit(projectInfo.TestModule); TestCoverageInjector.Instance.InjectAssemblyReferences(projectInfo.TestModule); using var ms = new MemoryStream(); projectInfo.TestModule.Write(ms); projectInfo.TestModule.Dispose(); File.WriteAllBytes(projectInfo.TestModule.FileName, ms.ToArray()); foreach (var assembly in projectInfo.DependencyAssemblies) { TestCoverageInjector.Instance.InjectAssemblyReferences(assembly.Module); TestCoverageInjector.Instance.InjectTargetCoverage(assembly.Module); assembly.Flush(); assembly.Dispose(); } if (projectInfo.TestFramework == TestFramework.XUnit) { var testDirectory = new FileInfo(projectInfo.TestModule.FileName).Directory; var xunitConfigFileName = Path.Combine(testDirectory.FullName, "xunit.runner.json"); var xunitCoverageSettings = JObject.FromObject(new { parallelizeTestCollections = false }); if (!File.Exists(xunitConfigFileName)) { File.WriteAllText(xunitConfigFileName, xunitCoverageSettings.ToString()); } else { var originalJsonConfig = JObject.Parse(File.ReadAllText(xunitConfigFileName)); originalJsonConfig.Merge(xunitCoverageSettings); File.WriteAllText(xunitConfigFileName, originalJsonConfig.ToString()); } } }
/// <summary> /// Returns information about the test project. /// </summary> /// <returns></returns> private TestProjectInfo GetTestProjectInfo(TestProjectDuplication duplication, IProjectInfo testProjectInfo) { var testFramework = GetTestFramework(testProjectInfo); // Read the test project into memory. var projectInfo = new TestProjectInfo { TestFramework = testFramework, TestModule = ModuleDefinition.ReadModule(duplication.TestProjectFile.FullFilePath()) }; // Foreach project reference load it in memory as an 'assembly mutator'. foreach (var projectReferencePath in duplication.TestProjectReferences) { var loadProjectReferenceModel = new AssemblyMutator(projectReferencePath.FullFilePath()); if (loadProjectReferenceModel.Types.Count > 0) { projectInfo.DependencyAssemblies.Add(loadProjectReferenceModel); } } return(projectInfo); }
/// <summary> /// Starts the mutation test session and returns the report with results. /// </summary> /// <param name="testProjectInfo"></param> /// <param name="testsPerMutation"></param> /// <param name="sessionProgressTracker"></param> /// <param name="coverageTestRunTime"></param> /// <param name="testProjectDuplicationPool"></param> /// <returns></returns> private TestProjectReportModel StartMutationTestSession(TestProjectInfo testProjectInfo, Dictionary <RegisteredCoverage, HashSet <string> > testsPerMutation, MutationSessionProgressTracker sessionProgressTracker, TimeSpan coverageTestRunTime, TestProjectDuplicationPool testProjectDuplicationPool) { // Generate the mutation test runs for the mutation session. var defaultMutationTestRunGenerator = new DefaultMutationTestRunGenerator(); var runs = defaultMutationTestRunGenerator.GenerateMutationTestRuns(testsPerMutation, testProjectInfo, _mutationLevel); // Double the time the code coverage took such that test runs have some time run their tests (needs to be in seconds). var maxTestDuration = TimeSpan.FromSeconds((coverageTestRunTime * 2).Seconds); var reportBuilder = new TestProjectReportModelBuilder(testProjectInfo.TestModule.Name); var allRunsStopwatch = new Stopwatch(); allRunsStopwatch.Start(); var mutationTestRuns = runs.ToList(); var totalRunsCount = mutationTestRuns.Count(); var mutationCount = mutationTestRuns.Sum(x => x.MutationCount); var completedRuns = 0; var failedRuns = 0; sessionProgressTracker.LogBeginTestSession(totalRunsCount, mutationCount, maxTestDuration); // Stores timed out mutations which will be excluded from test runs if they occur. // Timed out mutations will be removed because they can cause serious test delays. var timedOutMutations = new List <MutationVariantIdentifier>(); async Task RunTestRun(IMutationTestRun testRun) { var testProject = testProjectDuplicationPool.AcquireTestProject(); try { testRun.InitializeMutations(testProject, timedOutMutations); var singRunsStopwatch = new Stopwatch(); singRunsStopwatch.Start(); var results = await testRun.RunMutationTestAsync(maxTestDuration, sessionProgressTracker, _testHostRunFactory, testProject, _testHostLogger); if (results != null) { foreach (var testResult in results) { // Store the timed out mutations such that they can be excluded. timedOutMutations.AddRange(testResult.GetTimedOutTests()); // For each mutation add it to the report builder. reportBuilder.AddTestResult(testResult.TestResults, testResult.Mutations, singRunsStopwatch.Elapsed); } singRunsStopwatch.Stop(); singRunsStopwatch.Reset(); } } catch (Exception e) { _testHostLogger.LogError(e, "The test process encountered an unexpected error."); sessionProgressTracker.Log( $"The test process encountered an unexpected error. Continuing without this test run. Please consider to submit an github issue. {e}", LogMessageType.Error); failedRuns += 1; } finally { lock (this) { completedRuns += 1; sessionProgressTracker.LogTestRunUpdate(completedRuns, totalRunsCount, failedRuns); } testProject.MarkAsFree(); } } var tasks = from testRun in mutationTestRuns select RunTestRun(testRun); Task.WaitAll(tasks.ToArray()); allRunsStopwatch.Stop(); var report = reportBuilder.Build(allRunsStopwatch.Elapsed, totalRunsCount); sessionProgressTracker.LogEndTestSession(allRunsStopwatch.Elapsed, completedRuns, mutationCount, report.ScorePercentage); return(report); }