public TestRunResult CaptureCoverage(bool cantUsePipe, bool cantUseUnloadAppDomain)
{
if (cantUseUnloadAppDomain)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. ");
return new TestRunResult() { Success = true };
}
if (cantUsePipe)
{
_logger.LogDebug("Target framework does not support NamedPipes. Stryker will use environment variables instead.");
}
if (_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) || _flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var collector = new CoverageCollector();
collector.SetLogger((message) => _logger.LogTrace(message));
collector.Init(!cantUsePipe);
var coverageEnvironment = collector.GetEnvironmentVariables();
var result = LaunchTestProcess(null, coverageEnvironment);
var data = collector.RetrieveCoverData();
var coveredMutants = data.Split(";")[0].Split(",", StringSplitOptions.RemoveEmptyEntries);
CoverageMutants.DeclareCoveredMutants(coveredMutants.Select(int.Parse));
return result;
}
return new TestRunResult(){Success = true};
}
private void SetupMockTimeOutTestRun(Mock <IVsTestConsoleWrapper> mockVsTest, IReadOnlyDictionary <string, string> results, string timeoutTest, EventWaitHandle endProcess)
{
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
It.IsAny <IEnumerable <TestCase> >(),
It.IsAny <string>(),
It.IsAny <ITestRunEventsHandler>(),
It.IsAny <ITestHostLauncher>())).Callback(
(IEnumerable <TestCase> sources, string settings, ITestRunEventsHandler testRunEvents,
ITestHostLauncher host) =>
{
var collector = new CoverageCollector();
var start = new TestSessionStartArgs
{
Configuration = settings
};
var mock = new Mock <IDataCollectionSink>(MockBehavior.Loose);
TestCase timeOutTestCase = null;
collector.Initialize(mock.Object);
collector.TestSessionStart(start);
var mutants = collector.MutantList;
if (!results.ContainsKey(mutants))
{
throw new ArgumentException($"Unexpected mutant run {mutants}");
}
var tests = sources.ToList();
var data = results[mutants].Split(',').Select(e => e.Split('=')).ToList();
if (data.Count != tests.Count)
{
throw new ArgumentException($"Invalid number of tests for mutant run {mutants}: found {tests.Count}, expected {data.Count}");
}
var runResults = new List <TestResult>(data.Count);
foreach (var strings in data)
{
var matchingTest = tests.FirstOrDefault(t => t.FullyQualifiedName == strings[0]);
if (matchingTest == null)
{
throw new ArgumentException($"Test {strings[0]} not run for mutant {mutants}.");
}
if (matchingTest.FullyQualifiedName == timeoutTest)
{
timeOutTestCase = matchingTest;
}
var result = new TestResult(matchingTest)
{
Outcome = strings[1] == "F" ? TestOutcome.Failed : TestOutcome.Passed, ComputerName = "."
};
runResults.Add(result);
}
// setup a normal test run
MoqTestRun(testRunEvents, runResults, timeOutTestCase);
collector.TestSessionEnd(new TestSessionEndArgs());
endProcess.Set();
});
}
private string GenerateRunSettings(int?timeout, bool forCoverage)
{
var targetFramework = _projectInfo.TestProjectAnalyzerResult.TargetFramework;
var targetFrameworkVersion = Regex.Replace(targetFramework, @"[^.\d]", "");
switch (targetFramework)
{
case string s when s.Contains("netcoreapp"):
targetFrameworkVersion = $".NETCoreApp,Version = v{targetFrameworkVersion}";
break;
case string s when s.Contains("netstandard"):
throw new ApplicationException("Unsupported targetframework detected. A unit test project cannot be netstandard!: " + targetFramework);
default:
targetFrameworkVersion = $"Framework40";
break;
}
var needCoverage = forCoverage && NeedCoverage();
var dataCollectorSettings = needCoverage ? CoverageCollector.GetVsTestSettings() : "";
var settingsForCoverage = string.Empty;
if (needCoverage)
{
if (_testFramework.HasFlag(TestFramework.nUnit))
{
settingsForCoverage = "<CollectDataForEachTestSeparately>true</CollectDataForEachTestSeparately>";
}
if (_testFramework.HasFlag(TestFramework.xUnit))
{
settingsForCoverage += "<DisableParallelization>true</DisableParallelization>";
}
}
var timeoutSettings = timeout.HasValue ? $"<TestSessionTimeout>{timeout}</TestSessionTimeout>" : "";
var runSettings =
$@"<RunSettings>
<RunConfiguration>
<MaxCpuCount>{_options.ConcurrentTestrunners}</MaxCpuCount>
<TargetFrameworkVersion>{targetFrameworkVersion}</TargetFrameworkVersion>
{timeoutSettings}
{settingsForCoverage}
</RunConfiguration>{dataCollectorSettings}
</RunSettings>";
_logger.LogDebug("VsTest runsettings set to: {0}", runSettings);
return(runSettings);
}
private string GenerateRunSettings(int?timeout, bool forMutantTesting, bool forCoverage, Dictionary <int, IList <string> > mutantTestsMap)
{
var projectAnalyzerResult = _projectInfo.TestProjectAnalyzerResults.FirstOrDefault();
var targetFramework = projectAnalyzerResult.TargetFramework;
var targetFrameworkVersion = projectAnalyzerResult.TargetFrameworkVersion;
string targetFrameworkVersionString;
switch (targetFramework)
{
case Initialisation.Framework.NetCore:
targetFrameworkVersionString = $".NETCoreApp,Version=v{targetFrameworkVersion}";
break;
case Initialisation.Framework.NetStandard:
throw new StrykerInputException("Unsupported targetframework detected. A unit test project cannot be netstandard!: " + targetFramework);
default:
targetFrameworkVersionString = $".NETFramework,Version=v{targetFrameworkVersion.ToString(2)}";
break;
}
var needCoverage = forCoverage && NeedCoverage();
var dataCollectorSettings = (forMutantTesting || forCoverage) ? CoverageCollector.GetVsTestSettings(needCoverage, mutantTestsMap, CodeInjection.HelperNamespace) : "";
var settingsForCoverage = string.Empty;
if (_testFramework.HasFlag(TestFramework.nUnit))
{
settingsForCoverage = "<CollectDataForEachTestSeparately>true</CollectDataForEachTestSeparately>";
}
if (_testFramework.HasFlag(TestFramework.xUnit))
{
settingsForCoverage += "<DisableParallelization>true</DisableParallelization>";
}
var timeoutSettings = timeout.HasValue ? $"<TestSessionTimeout>{timeout}</TestSessionTimeout>" + Environment.NewLine : string.Empty;
// we need to block parallel run to capture coverage and when testing multiple mutants in a single run
var optionsConcurrentTestrunners = (forCoverage || !_options.Optimizations.HasFlag(OptimizationFlags.DisableTestMix)) ? 1 : _options.ConcurrentTestrunners;
var runSettings =
$@"<RunSettings>
<RunConfiguration>
{(targetFramework == Initialisation.Framework.NetClassic ? "<DisableAppDomain>true</DisableAppDomain>" : "")}
<MaxCpuCount>{optionsConcurrentTestrunners}</MaxCpuCount>
<TargetFrameworkVersion>{targetFrameworkVersionString}</TargetFrameworkVersion>{timeoutSettings}{settingsForCoverage}
<DesignMode>false</DesignMode>
<BatchSize>1</BatchSize>
</RunConfiguration>{dataCollectorSettings}
</RunSettings>";
_logger.LogDebug("VsTest run settings set to: {0}", runSettings);
return(runSettings);
}
private string GenerateRunSettings(int?timeout, bool forCoverage)
{
var targetFramework = _projectInfo.TestProjectAnalyzerResults.FirstOrDefault().TargetFramework;
var targetFrameworkVersion = _projectInfo.TestProjectAnalyzerResults.FirstOrDefault().TargetFrameworkVersion;
string targetFrameworkVersionString;
switch (targetFramework)
{
case Initialisation.Framework.NetCore:
targetFrameworkVersionString = $".NETCoreApp,Version = v{targetFrameworkVersion}";
break;
case Initialisation.Framework.NetStandard:
throw new StrykerInputException("Unsupported targetframework detected. A unit test project cannot be netstandard!: " + targetFramework);
default:
targetFrameworkVersionString = $".NETFramework,Version = v{targetFrameworkVersion}";
break;
}
var needCoverage = forCoverage && NeedCoverage();
var dataCollectorSettings = needCoverage ? CoverageCollector.GetVsTestSettings() : "";
var settingsForCoverage = string.Empty;
if (needCoverage)
{
if (_testFramework.HasFlag(TestFramework.nUnit))
{
settingsForCoverage = "<CollectDataForEachTestSeparately>true</CollectDataForEachTestSeparately>";
}
if (_testFramework.HasFlag(TestFramework.xUnit))
{
settingsForCoverage += "<DisableParallelization>true</DisableParallelization>";
}
}
var timeoutSettings = timeout.HasValue ? $"<TestSessionTimeout>{timeout}</TestSessionTimeout>" : "";
var runSettings =
$@"<RunSettings>
<RunConfiguration>
<MaxCpuCount>{_options.ConcurrentTestrunners}</MaxCpuCount>
<TargetFrameworkVersion>{targetFrameworkVersionString}</TargetFrameworkVersion>
{timeoutSettings}
{settingsForCoverage}
</RunConfiguration>{dataCollectorSettings}
</RunSettings>";
_logger.LogDebug($"{RunnerId}: VsTest runsettings set to: {runSettings}");
return(runSettings);
}
public void ProperlyCaptureParams()
{
var collector = new CoverageCollector();
var start = new TestSessionStartArgs
{
Configuration = CoverageCollector.GetVsTestSettings(true, null, "Stryker.Core.UnitTest.TestRunners")
};
var mock = new Mock <IDataCollectionSink>(MockBehavior.Loose);
collector.Initialize(mock.Object);
collector.TestSessionStart(start);
collector.TestCaseStart(new TestCaseStartArgs(new TestCase("theTest", new Uri("xunit://"), "source.cs")));
MutantControl.CaptureCoverage.ShouldBeTrue();
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseUnloadAppDomain, bool cantUsePipe)
{
if (!_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) &&
!_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
return(new TestRunResult(true));
}
if (cantUseUnloadAppDomain)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. ");
return(new TestRunResult(true));
}
if (cantUsePipe)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support pipes. ");
return(new TestRunResult(true));
}
var collector = new CoverageCollector();
collector.SetLogger(message => _logger.LogTrace(message));
var coverageEnvironment = new Dictionary <string, string>();
coverageEnvironment["Coverage"] = $"pipe:{_server.PipeName}";
var result = LaunchTestProcess(null, coverageEnvironment);
if (!WaitOnLck(_lck, () => _lastMessage != null, 5000))
{
// Failed to retrieve coverage data for testCase
return(null);
}
var testedMutant = _lastMessage.Split(";")[0].Split(",").Select(int.Parse).ToList();
foreach (var mutant in mutants)
{
mutant.CoveringTests = testedMutant.Contains(mutant.Id) ? TestListDescription.EveryTest() : new TestListDescription(null);
}
return(result);
}
public TestRunResult CaptureCoverage()
{
if (_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants))
{
var collector = new CoverageCollector();
collector.SetLogger((message) => Logger.LogTrace(message));
collector.Init(true);
var coverageEnvironment = collector.GetEnvironmentVariables();
var result = LaunchTestProcess(null, coverageEnvironment);
var data = collector.RetrieveCoverData("full");
CoverageMutants.DeclareCoveredMutants(data.Split(",").Select(int.Parse));
return(result);
}
else
{
return(LaunchTestProcess(null, null));
}
}
private string GenerateRunSettings(int?timeout, bool forMutantTesting, bool forCoverage, Dictionary <int, IList <string> > mutantTestsMap)
{
var projectAnalyzerResult = _projectInfo.TestProjectAnalyzerResults.FirstOrDefault();
var targetFramework = projectAnalyzerResult.TargetFramework;
var needCoverage = forCoverage && NeedCoverage();
var dataCollectorSettings = (forMutantTesting || forCoverage) ? CoverageCollector.GetVsTestSettings(needCoverage, mutantTestsMap, CodeInjection.HelperNamespace) : "";
var settingsForCoverage = string.Empty;
if (_testFramework.HasFlag(TestFramework.nUnit))
{
settingsForCoverage = "<CollectDataForEachTestSeparately>true</CollectDataForEachTestSeparately>";
}
if (_testFramework.HasFlag(TestFramework.xUnit))
{
settingsForCoverage += "<DisableParallelization>true</DisableParallelization>";
}
var timeoutSettings = timeout.HasValue ? $"<TestSessionTimeout>{timeout}</TestSessionTimeout>" + Environment.NewLine : string.Empty;
// we need to block parallel run to capture coverage and when testing multiple mutants in a single run
var optionsConcurrentTestrunners = (forCoverage || !_options.Optimizations.HasFlag(OptimizationFlags.DisableTestMix)) ? 1 : _options.ConcurrentTestrunners;
var runSettings =
$@"<RunSettings>
<RunConfiguration>
{(targetFramework == Framework.DotNetClassic ? "<DisableAppDomain>true</DisableAppDomain>" : "")}
<MaxCpuCount>{optionsConcurrentTestrunners}</MaxCpuCount>
{timeoutSettings}
{settingsForCoverage}
<DesignMode>false</DesignMode>
<BatchSize>1</BatchSize>
</RunConfiguration>
{dataCollectorSettings}
</RunSettings>";
_logger.LogDebug("VsTest run settings set to: {0}", runSettings);
return(runSettings);
}
public void ProperlySelectMutant()
{
var collector = new CoverageCollector();
var mutantMap = new Dictionary <int, IList <string> >()
{
[0] = new List <string>()
};
var start = new TestSessionStartArgs
{
Configuration = CoverageCollector.GetVsTestSettings(false, mutantMap, this.GetType().Namespace)
};
var mock = new Mock <IDataCollectionSink>(MockBehavior.Loose);
collector.Initialize(mock.Object);
collector.TestSessionStart(start);
collector.TestCaseStart(new TestCaseStartArgs(new TestCase("theTest", new Uri("xunit://"), "source.cs")));
MutantControl.ActiveMutant.ShouldBe(0);
}