public TestRunResult CaptureCoverage(bool cantUsePipe, bool cantUseUnloadAppDomain)
{
if (cantUseUnloadAppDomain)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. ");
return new TestRunResult() { Success = true };
}
if (cantUsePipe)
{
_logger.LogDebug("Target framework does not support NamedPipes. Stryker will use environment variables instead.");
}
if (_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) || _flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var collector = new CoverageCollector();
collector.SetLogger((message) => _logger.LogTrace(message));
collector.Init(!cantUsePipe);
var coverageEnvironment = collector.GetEnvironmentVariables();
var result = LaunchTestProcess(null, coverageEnvironment);
var data = collector.RetrieveCoverData();
var coveredMutants = data.Split(";")[0].Split(",", StringSplitOptions.RemoveEmptyEntries);
CoverageMutants.DeclareCoveredMutants(coveredMutants.Select(int.Parse));
return result;
}
return new TestRunResult(){Success = true};
}
public TestRunResult CaptureCoverage()
{
TestRunResult result;
var needCoverage = _flags.HasFlag(OptimizationFlags.CoverageBasedTest) || _flags.HasFlag(OptimizationFlags.SkipUncoveredMutants);
if (needCoverage && _flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
{
return(CaptureCoveragePerIsolatedTests());
}
var runner = TakeRunner();
try
{
if (needCoverage)
{
result = runner.CaptureCoverage();
_coverage = runner.CoverageMutants;
}
else
{
result = runner.RunAll(null, null);
}
}
finally
{
ReturnRunner(runner);
}
return(result);
}
public TestRunResult CaptureCoverage(bool cantUsePipe, bool cantUseUnloadAppDomain)
{
var needCoverage = _flags.HasFlag(OptimizationFlags.CoverageBasedTest) || _flags.HasFlag(OptimizationFlags.SkipUncoveredMutants);
if (needCoverage && _flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
{
return(CaptureCoveragePerIsolatedTests(cantUsePipe));
}
var runner = TakeRunner();
TestRunResult result;
try
{
if (needCoverage)
{
result = runner.CaptureCoverage(cantUsePipe, cantUseUnloadAppDomain);
CoverageMutants = runner.CoverageMutants;
}
else
{
result = runner.RunAll(null, null);
}
}
finally
{
ReturnRunner(runner);
}
return(result);
}
public VsTestRunner(
StrykerOptions options,
OptimizationFlags flags,
ProjectInfo projectInfo,
ICollection <TestCase> testCasesDiscovered,
TestCoverageInfos mappingInfos,
IFileSystem fileSystem = null,
IVsTestHelper helper = null,
ILogger logger = null,
IVsTestConsoleWrapper wrapper = null,
Func <IDictionary <string, string>, int, IStrykerTestHostLauncher> hostBuilder = null)
{
_logger = logger ?? ApplicationLogging.LoggerFactory.CreateLogger <VsTestRunner>();
_fileSystem = fileSystem ?? new FileSystem();
_options = options;
_flags = flags;
_projectInfo = projectInfo;
_hostBuilder = hostBuilder ?? ((dico, id) => new StrykerVsTestHostLauncher(dico, id));
SetListOfTests(testCasesDiscovered);
_ownHelper = helper == null;
_vsTestHelper = helper ?? new VsTestHelper();
CoverageMutants = mappingInfos ?? new TestCoverageInfos();
_vsTestConsole = wrapper ?? PrepareVsTestConsole();
_id = _count++;
if (testCasesDiscovered != null)
{
_discoveredTests = testCasesDiscovered;
DetectTestFramework(testCasesDiscovered);
}
InitializeVsTestConsole();
_coverageEnvironment = new Dictionary <string, string>
{
{ CoverageCollector.ModeEnvironmentVariable, flags.HasFlag(OptimizationFlags.UseEnvVariable) ? CoverageCollector.EnvMode : CoverageCollector.PipeMode }
};
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseUnloadAppDomain, bool cantUsePipe)
{
if (!_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) &&
!_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
return(new TestRunResult(true));
}
if (cantUseUnloadAppDomain)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support 'AppDomain'. ");
return(new TestRunResult(true));
}
if (cantUsePipe)
{
_logger.LogWarning("Can't capture the test coverage as the target framework does not support pipes. ");
return(new TestRunResult(true));
}
var collector = new CoverageCollector();
collector.SetLogger(message => _logger.LogTrace(message));
var coverageEnvironment = new Dictionary <string, string>();
coverageEnvironment["Coverage"] = $"pipe:{_server.PipeName}";
var result = LaunchTestProcess(null, coverageEnvironment);
if (!WaitOnLck(_lck, () => _lastMessage != null, 5000))
{
// Failed to retrieve coverage data for testCase
return(null);
}
var testedMutant = _lastMessage.Split(";")[0].Split(",").Select(int.Parse).ToList();
foreach (var mutant in mutants)
{
mutant.CoveringTests = testedMutant.Contains(mutant.Id) ? TestListDescription.EveryTest() : new TestListDescription(null);
}
return(result);
}
public TestRunResult CaptureCoverage()
{
if (_flags.HasFlag(OptimizationFlags.SkipUncoveredMutants) || _flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var collector = new CoverageCollector();
collector.SetLogger((message) => _logger.LogTrace(message));
collector.Init(true);
var coverageEnvironment = collector.GetEnvironmentVariables();
var result = LaunchTestProcess(null, coverageEnvironment);
var data = collector.RetrieveCoverData("full");
CoverageMutants.DeclareCoveredMutants(data.Split(";")[0].Split(",").Select(int.Parse));
return(result);
}
else
{
return(LaunchTestProcess(null, null));
}
}
public TestRunResult RunAll(int?timeoutMs, int?mutationId)
{
var envVars = new Dictionary <string, string>();
if (mutationId != null)
{
envVars["ActiveMutation"] = mutationId.ToString();
}
var testCases = (mutationId == null || !_flags.HasFlag(OptimizationFlags.CoverageBasedTest)) ? null : CoverageMutants.GetTests <TestCase>(mutationId.Value);
if (testCases == null)
{
_logger.LogDebug($"Runner {_id}: Testing {mutationId} against all tests.");
}
else
{
_logger.LogDebug($"Runner {_id}: Testing {mutationId} against:{string.Join(", ", testCases.Select(x => x.FullyQualifiedName))}.");
}
return(RunVsTest(testCases, timeoutMs, envVars));
}
public TestRunResult RunAll(int?timeoutMs, IReadOnlyMutant mutant)
{
var envVars = new Dictionary <string, string>();
if (mutant != null)
{
envVars["ActiveMutation"] = mutant.Id.ToString();
}
if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest) && mutant != null && (mutant.CoveringTest == null || mutant.CoveringTest.Count == 0))
{
return(new TestRunResult {
ResultMessage = "Not covered by any test", Success = true
});
}
IEnumerable <TestCase> testCases = null;
// if we optimize the number of test to run
if (mutant != null && _flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
// we must run all tests if the mutants needs it (static) except when coverage has been captured by isolated test
testCases = (mutant.MustRunAllTests && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
? null : _discoveredTests.Where(t => mutant.CoveringTest.ContainsKey(t.Id.ToString())).ToList();
if (testCases == null)
{
_logger.LogDebug($"Runner {_id}: Testing {mutant} against all tests.");
}
else
{
_logger.LogDebug($"Runner {_id}: Testing {mutant} against:{string.Join(", ", testCases.Select(x => x.FullyQualifiedName))}.");
}
}
return(RunVsTest(testCases, timeoutMs, envVars));
}
public TestRunResult CaptureCoverage(IEnumerable <Mutant> mutants, bool cantUseAppDomain, bool cantUsePipe)
{
var needCoverage = _flags.HasFlag(OptimizationFlags.CoverageBasedTest) || _flags.HasFlag(OptimizationFlags.SkipUncoveredMutants);
if (needCoverage && _flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest))
{
return(CaptureCoveragePerIsolatedTests(mutants, cantUseAppDomain, cantUsePipe));
}
var runner = TakeRunner();
TestRunResult result;
try
{
result = needCoverage ? runner.CaptureCoverage(mutants, cantUseAppDomain, cantUsePipe) : runner.RunAll(null, null, null);
}
finally
{
ReturnRunner(runner);
}
return(result);
}
public VsTestRunner(StrykerOptions options,
OptimizationFlags flags,
ProjectInfo projectInfo,
ICollection <TestCase> testCasesDiscovered,
TestCoverageInfos mappingInfos,
IFileSystem fileSystem = null,
VsTestHelper helper = null)
{
_fileSystem = fileSystem ?? new FileSystem();
_options = options;
_flags = flags;
_projectInfo = projectInfo;
SetListOfTests(testCasesDiscovered);
_ownHelper = helper == null;
_vsTestHelper = helper ?? new VsTestHelper();
CoverageMutants = mappingInfos ?? new TestCoverageInfos();
_vsTestConsole = PrepareVsTestConsole();
_id = _count++;
InitializeVsTestConsole();
_coverageEnvironment = new Dictionary <string, string>
{
{ CoverageCollector.ModeEnvironmentVariable, flags.HasFlag(OptimizationFlags.UseEnvVariable) ? CoverageCollector.EnvMode : CoverageCollector.PipeMode }
};
}
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, IList <string> >();
ICollection <TestCase> testCases = null;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var needAll = false;
foreach (var mutant in mutants)
{
List <string> tests;
if ((mutant.IsStaticValue && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = null;
needAll = true;
}
else
{
tests = mutant.CoveringTests.GetList().Select(t => t.Guid).ToList();
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetList()).Distinct().Select(t => _discoveredTests.First(tc => tc.Id.ToString() == t.Guid)).ToList();
_logger.LogDebug($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases.Select(x => x.FullyQualifiedName)))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.NoTest(), "Mutants are not covered by any test!"));
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants.FirstOrDefault().Id, new List <string>());
}
}
var expectedTests = testCases?.Count ?? DiscoverNumberOfTests();
void HandleUpdate(IRunResults handler)
{
if (mutants == null)
{
return;
}
var handlerTestResults = handler.TestResults;
var tests = handlerTestResults.Count == DiscoverNumberOfTests()
? TestListDescription.EveryTest()
: new TestListDescription(handlerTestResults.Select(tr => (TestDescription)tr.TestCase));
var failedTest = new TestListDescription(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(tr => (TestDescription)tr.TestCase));
var testsInProgress = new TestListDescription(handler.TestsInTimeout?.Select(t => (TestDescription)t));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, testsInProgress);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
var testResults = RunTestSession(testCases, GenerateRunSettings(timeoutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
var resultAsArray = testResults.TestResults.ToArray();
var timeout = (!_aborted && resultAsArray.Length < expectedTests);
var ranTests = resultAsArray.Length == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(resultAsArray.Select(tr => (TestDescription)tr.TestCase));
var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(tr => (TestDescription)tr.TestCase).ToImmutableArray();
if (ranTests.Count == 0 && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0))
{
_logger.LogDebug($"{RunnerId}: Test session reports 0 result and 0 stuck tests.");
}
var message = string.Join(Environment.NewLine,
resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage))
.Select(tr => tr.ErrorMessage));
var failedTestsDescription = new TestListDescription(failedTests);
var timedOutTests = new TestListDescription(testResults.TestsInTimeout?.Select(t => (TestDescription)t));
return(timeout ? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message) : new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message));
}
