/// <summary>
/// Executes the mutation test session for the given test project.
/// Algorithm:
/// 1. Build project
/// 2. Calculate for each test which mutations they cover.
/// 3. Rebuild to remove injected code.
/// 4. Run test session.
/// 4a. Generate optimized test runs.
/// 5. Build report.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <TestProjectReportModel> Test(MutationSessionProgressTracker progressTracker,
CancellationToken cancellationToken = default)
{
// Build project
progressTracker.LogBeginPreBuilding();
var projectInfo = await BuildProject(progressTracker, _testProjectPath);
progressTracker.LogEndPreBuilding();
// Copy project N times
progressTracker.LogBeginProjectDuplication(_parallel);
var testProjectCopier = new TestProjectDuplicator(Directory.GetParent(projectInfo.AssemblyPath).FullName);
var duplications = testProjectCopier.MakeInitialCopies(projectInfo, _parallel);
// Begin code coverage on first project.
var duplicationPool = new TestProjectDuplicationPool(duplications);
var coverageProject = duplicationPool.TakeTestProject();
var coverageProjectInfo = GetTestProjectInfo(coverageProject, projectInfo);
// Measure the test coverage
progressTracker.LogBeginCoverage();
PrepareAssembliesForCodeCoverage(coverageProjectInfo);
var coverageTimer = new Stopwatch();
coverageTimer.Start();
var coverage = await RunCoverage(coverageProject.TestProjectFile.FullFilePath(), cancellationToken);
coverageTimer.Stop();
var timeout = _createTimeOut(coverageTimer);
GC.Collect();
GC.WaitForPendingFinalizers();
coverageProject.MarkAsFree();
// Start test session.
var testsPerMutation = GroupMutationsWithTests(coverage);
return(StartMutationTestSession(coverageProjectInfo, testsPerMutation, progressTracker,
timeout, duplicationPool));
}
/// <summary>
/// Starts the mutation test session and returns the report with results.
/// </summary>
/// <param name="testProjectInfo"></param>
/// <param name="testsPerMutation"></param>
/// <param name="sessionProgressTracker"></param>
/// <param name="coverageTestRunTime"></param>
/// <param name="testProjectDuplicationPool"></param>
/// <returns></returns>
private TestProjectReportModel StartMutationTestSession(TestProjectInfo testProjectInfo,
Dictionary <RegisteredCoverage, HashSet <string> > testsPerMutation,
MutationSessionProgressTracker sessionProgressTracker, TimeSpan coverageTestRunTime,
TestProjectDuplicationPool testProjectDuplicationPool)
{
// Generate the mutation test runs for the mutation session.
var defaultMutationTestRunGenerator = new DefaultMutationTestRunGenerator();
var runs = defaultMutationTestRunGenerator.GenerateMutationTestRuns(testsPerMutation, testProjectInfo,
_mutationLevel);
// Double the time the code coverage took such that test runs have some time run their tests (needs to be in seconds).
var maxTestDuration = TimeSpan.FromSeconds((coverageTestRunTime * 2).Seconds);
var reportBuilder = new TestProjectReportModelBuilder(testProjectInfo.TestModule.Name);
var allRunsStopwatch = new Stopwatch();
allRunsStopwatch.Start();
var mutationTestRuns = runs.ToList();
var totalRunsCount = mutationTestRuns.Count();
var mutationCount = mutationTestRuns.Sum(x => x.MutationCount);
var completedRuns = 0;
var failedRuns = 0;
sessionProgressTracker.LogBeginTestSession(totalRunsCount, mutationCount, maxTestDuration);
// Stores timed out mutations which will be excluded from test runs if they occur.
// Timed out mutations will be removed because they can cause serious test delays.
var timedOutMutations = new List <MutationVariantIdentifier>();
async Task RunTestRun(IMutationTestRun testRun)
{
var testProject = testProjectDuplicationPool.AcquireTestProject();
try
{
testRun.InitializeMutations(testProject, timedOutMutations);
var singRunsStopwatch = new Stopwatch();
singRunsStopwatch.Start();
var results = await testRun.RunMutationTestAsync(maxTestDuration, sessionProgressTracker,
_testHostRunFactory, testProject, _testHostLogger);
if (results != null)
{
foreach (var testResult in results)
{
// Store the timed out mutations such that they can be excluded.
timedOutMutations.AddRange(testResult.GetTimedOutTests());
// For each mutation add it to the report builder.
reportBuilder.AddTestResult(testResult.TestResults, testResult.Mutations,
singRunsStopwatch.Elapsed);
}
singRunsStopwatch.Stop();
singRunsStopwatch.Reset();
}
}
catch (Exception e)
{
_testHostLogger.LogError(e, "The test process encountered an unexpected error.");
sessionProgressTracker.Log(
$"The test process encountered an unexpected error. Continuing without this test run. Please consider to submit an github issue. {e}",
LogMessageType.Error);
failedRuns += 1;
}
finally
{
lock (this)
{
completedRuns += 1;
sessionProgressTracker.LogTestRunUpdate(completedRuns, totalRunsCount, failedRuns);
}
testProject.MarkAsFree();
}
}
var tasks = from testRun in mutationTestRuns select RunTestRun(testRun);
Task.WaitAll(tasks.ToArray());
allRunsStopwatch.Stop();
var report = reportBuilder.Build(allRunsStopwatch.Elapsed, totalRunsCount);
sessionProgressTracker.LogEndTestSession(allRunsStopwatch.Elapsed, completedRuns, mutationCount,
report.ScorePercentage);
return(report);
}
