private TestRunResult RunTestSession(IList <Mutant> mutantsToTest, ITimeoutValueCalculator timeoutMs,
TestUpdateHandler updateHandler, bool forceSingle)
{
TestRunResult result;
Logger.LogTrace($"Testing {string.Join(" ,", mutantsToTest.Select(x => x.DisplayName))}.");
if (forceSingle && mutantsToTest.Count > 1)
{
foreach (var mutant in mutantsToTest)
{
var localResult = TestRunner.TestMultipleMutants(timeoutMs, new[] { mutant }, updateHandler);
mutant.AnalyzeTestRun(localResult.FailingTests, localResult.RanTests, localResult.TimedOutTests);
}
return(new TestRunResult(true));
}
else
{
result = TestRunner.TestMultipleMutants(timeoutMs, mutantsToTest.ToList(), updateHandler);
if (updateHandler == null)
{
foreach (var mutant in mutantsToTest)
{
mutant.AnalyzeTestRun(result.FailingTests, result.RanTests, result.TimedOutTests);
}
}
}
return(result);
}
public void TheDataStreamTest()
{
var fileWriter = new StreamWriter("SimulatedDataSourceTest.csv");
fileWriter.WriteLine("Registered,1,8/15/2017 7:00:00 AM,FN_F_1,LN_1,F,16");
fileWriter.WriteLine("Registered,2,8/15/2017 7:00:00 AM,FN_M_2,LN_2,M,56");
fileWriter.WriteLine("---");
fileWriter.WriteLine("Started,1,8/15/2017 7:02:00 AM,8/15/2017 7:02:00 AM");
fileWriter.WriteLine("---");
fileWriter.WriteLine("OnCourse,1,8/15/2017 7:02:30 AM,276.098203867211");
fileWriter.WriteLine("Started,2,8/15/2017 7:02:30 AM,8/15/2017 7:02:30 AM");
fileWriter.WriteLine("---");
fileWriter.WriteLine("Finished,1,8/15/2017 7:03:00 AM,8/15/2017 7:03:00 AM");
fileWriter.Close();
var testUpdateHandler = new TestUpdateHandler();
var ds = new SimulatedDataSource
{
InputFilename = "SimulatedDataSourceTest.csv",
Handler = testUpdateHandler
};
ds.Start();
// Wait for something about 1/2 second
Thread.Sleep(500);
// Two messages should have come in
Assert.AreEqual(2, testUpdateHandler.NumberRecieved);
Assert.AreEqual(AthleteRaceStatus.Registered, testUpdateHandler[0].UpdateType);
Assert.AreEqual(AthleteRaceStatus.Registered, testUpdateHandler[1].UpdateType);
// Wait for something less than a second
Thread.Sleep(1000);
// Another message should have some in
Assert.AreEqual(3, testUpdateHandler.NumberRecieved);
Assert.AreEqual(AthleteRaceStatus.Started, testUpdateHandler[2].UpdateType);
// Wait for something less than a second
Thread.Sleep(2000);
// Another message should have some in
Assert.AreEqual(6, testUpdateHandler.NumberRecieved);
Assert.AreEqual(AthleteRaceStatus.OnCourse, testUpdateHandler[3].UpdateType);
Assert.AreEqual(AthleteRaceStatus.Started, testUpdateHandler[4].UpdateType);
Assert.AreEqual(AthleteRaceStatus.Finished, testUpdateHandler[5].UpdateType);
ds.Stop();
Thread.Sleep(1000);
Assert.AreEqual(6, testUpdateHandler.NumberRecieved);
}
public void Test(IList <Mutant> mutantsToTest, int timeoutMs, TestUpdateHandler updateHandler)
{
var forceSingle = false;
while (mutantsToTest.Any())
{
var result = RunTestSession(mutantsToTest, timeoutMs, updateHandler, forceSingle);
Logger.LogDebug(
$"Test run for {string.Join(" ,", mutantsToTest.Select(x => x.DisplayName))} is {(result.FailingTests.Count == 0 ? "success" : "failed")} with output: {result.ResultMessage}");
var remainingMutants = mutantsToTest.Where((m) => m.ResultStatus == MutantStatus.NotRun).ToList();
if (remainingMutants.Count == mutantsToTest.Count)
{
// the test failed to get any conclusive results
if (!result.SessionTimedOut)
{
// something bad happened.
Logger.LogError($"Stryker failed to test {remainingMutants.Count} mutant(s).");
return;
}
// test session's results have been corrupted by the time out
// we retry and run tests one by one, if necessary
if (remainingMutants.Count == 1)
{
// only one mutant was tested, we mark it as timeout.
remainingMutants[0].ResultStatus = MutantStatus.Timeout;
remainingMutants.Clear();
}
else
{
forceSingle = true;
}
}
if (remainingMutants.Any())
{
Logger.LogDebug("Not all mutants were tested.");
}
mutantsToTest = remainingMutants;
}
}
public TestRunResult RunAll(int?timeoutMs, Mutant mutant, TestUpdateHandler update)
{
var envVars = mutant == null ? null :
new Dictionary <string, string>
{
{ "ActiveMutation", mutant.Id.ToString() }
};
try
{
var result = LaunchTestProcess(timeoutMs, envVars);
update?.Invoke(new[] { mutant }, result.RanTests, result.FailingTests, result.TimedOutTests);
return(result);
}
catch (OperationCanceledException)
{
var emptyList = new TestListDescription(null);
if (mutant != null)
{
mutant.ResultStatus = MutantStatus.Timeout;
}
return(TestRunResult.TimedOut(emptyList, emptyList, TestListDescription.EveryTest(), "time out"));
}
}
public TestRunResult RunAll(int?timeoutMs, Mutant mutant, TestUpdateHandler update)
{
return(TestMultipleMutants(timeoutMs, mutant == null ? null : new List <Mutant> {
mutant
}, update));
}
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var runner = TakeRunner();
try
{
return(mutants == null?runner.RunAll(timeoutMs, null, update) : runner.TestMultipleMutants(timeoutMs, mutants, update));
}
finally
{
ReturnRunner(runner);
}
}
public TestRunResult TestMultipleMutants(int?timeoutMs, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, IList <string> >();
ICollection <TestCase> testCases = null;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_flags.HasFlag(OptimizationFlags.CoverageBasedTest))
{
var needAll = false;
foreach (var mutant in mutants)
{
List <string> tests;
if ((mutant.IsStaticValue && !_flags.HasFlag(OptimizationFlags.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = null;
needAll = true;
}
else
{
tests = mutant.CoveringTests.GetList().Select(t => t.Guid).ToList();
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetList()).Distinct().Select(t => _discoveredTests.First(tc => tc.Id.ToString() == t.Guid)).ToList();
_logger.LogDebug($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases.Select(x => x.FullyQualifiedName)))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestListDescription.NoTest(), TestListDescription.NoTest(), TestListDescription.NoTest(), "Mutants are not covered by any test!"));
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants.FirstOrDefault().Id, new List <string>());
}
}
var expectedTests = testCases?.Count ?? DiscoverNumberOfTests();
void HandleUpdate(IRunResults handler)
{
if (mutants == null)
{
return;
}
var handlerTestResults = handler.TestResults;
var tests = handlerTestResults.Count == DiscoverNumberOfTests()
? TestListDescription.EveryTest()
: new TestListDescription(handlerTestResults.Select(tr => (TestDescription)tr.TestCase));
var failedTest = new TestListDescription(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(tr => (TestDescription)tr.TestCase));
var testsInProgress = new TestListDescription(handler.TestsInTimeout?.Select(t => (TestDescription)t));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, testsInProgress);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
var testResults = RunTestSession(testCases, GenerateRunSettings(timeoutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
var resultAsArray = testResults.TestResults.ToArray();
var timeout = (!_aborted && resultAsArray.Length < expectedTests);
var ranTests = resultAsArray.Length == DiscoverNumberOfTests() ? TestListDescription.EveryTest() : new TestListDescription(resultAsArray.Select(tr => (TestDescription)tr.TestCase));
var failedTests = resultAsArray.Where(tr => tr.Outcome == TestOutcome.Failed).Select(tr => (TestDescription)tr.TestCase).ToImmutableArray();
if (ranTests.Count == 0 && (testResults.TestsInTimeout == null || testResults.TestsInTimeout.Count == 0))
{
_logger.LogDebug($"{RunnerId}: Test session reports 0 result and 0 stuck tests.");
}
var message = string.Join(Environment.NewLine,
resultAsArray.Where(tr => !string.IsNullOrWhiteSpace(tr.ErrorMessage))
.Select(tr => tr.ErrorMessage));
var failedTestsDescription = new TestListDescription(failedTests);
var timedOutTests = new TestListDescription(testResults.TestsInTimeout?.Select(t => (TestDescription)t));
return(timeout ? TestRunResult.TimedOut(ranTests, failedTestsDescription, timedOutTests, message) : new TestRunResult(ranTests, failedTestsDescription, timedOutTests, message));
}
private TestRunResult RunTestSession(IList <Mutant> mutantsToTest, int timeoutMs, TestUpdateHandler updateHandler,
bool forceSingle)
{
TestRunResult result = null;
Logger.LogTrace($"Testing {string.Join(" ,", mutantsToTest.Select(x => x.DisplayName))}.");
if (TestRunner is IMultiTestRunner multi && !forceSingle)
{
result = multi.TestMultipleMutants(timeoutMs, mutantsToTest.ToList(), updateHandler);
}
public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update)
{
return(TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List <Mutant> {
activeMutant
}, update));
}
public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList <Mutant> mutants, TestUpdateHandler update)
{
var mutantTestsMap = new Dictionary <int, ITestGuids>();
var needAll = true;
ICollection <Guid> testCases;
int?timeOutMs = timeoutCalc?.DefaultTimeout;
if (mutants != null)
{
// if we optimize the number of tests to run
if (_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest))
{
needAll = false;
foreach (var mutant in mutants)
{
ITestListDescription tests;
if ((mutant.IsStaticValue && !_options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest)) || mutant.MustRunAgainstAllTests)
{
tests = TestsGuidList.EveryTest();
needAll = true;
}
else
{
tests = mutant.CoveringTests;
}
mutantTestsMap.Add(mutant.Id, tests);
}
testCases = needAll ? null : mutants.SelectMany(m => m.CoveringTests.GetGuids()).ToList();
_logger.LogTrace($"{RunnerId}: Testing [{string.Join(',', mutants.Select(m => m.DisplayName))}] " +
$"against {(testCases == null ? "all tests." : string.Join(", ", testCases))}.");
if (testCases?.Count == 0)
{
return(new TestRunResult(TestsGuidList.NoTest(), TestsGuidList.NoTest(), TestsGuidList.NoTest(), "Mutants are not covered by any test!", TimeSpan.Zero));
}
if (timeoutCalc != null && testCases != null)
{
// compute time out
var duration = (int)testCases.Select(id => _vsTests[id].InitialRunTime.TotalMilliseconds).Sum();
timeOutMs = timeoutCalc.CalculateTimeoutValue(duration);
}
}
else
{
if (mutants.Count > 1)
{
throw new GeneralStrykerException("Internal error: trying to test multiple mutants simultaneously without 'perTest' coverage analysis.");
}
mutantTestsMap.Add(mutants[0].Id, TestsGuidList.EveryTest());
testCases = null;
}
}
else
{
testCases = null;
}
var numberTestCases = testCases == null ? 0 : testCases.Count;
var expectedTests = needAll ? DiscoverTests().Count : numberTestCases;
void HandleUpdate(IRunResults handler)
{
var handlerTestResults = handler.TestResults;
if (mutants == null)
{
return;
}
var tests = handlerTestResults.Count == DiscoverTests().Count
? (ITestGuids)TestsGuidList.EveryTest()
: new WrappedGuidsEnumeration(handlerTestResults.Select(t => t.TestCase.Id));
var failedTest = new WrappedGuidsEnumeration(handlerTestResults.Where(tr => tr.Outcome == TestOutcome.Failed)
.Select(t => t.TestCase.Id));
var timedOutTest = new WrappedGuidsEnumeration(handler.TestsInTimeout?.Select(t => t.Id));
var remainingMutants = update?.Invoke(mutants, failedTest, tests, timedOutTest);
if (handlerTestResults.Count >= expectedTests || remainingMutants != false || _aborted)
{
return;
}
// all mutants status have been resolved, we can stop
_logger.LogDebug($"{RunnerId}: Each mutant's fate has been established, we can stop.");
_vsTestConsole.CancelTestRun();
_aborted = true;
}
_logger.LogDebug("Using {0} ms as testrun timeout", timeOutMs.ToString() ?? "no");
var testResults = RunTestSession(mutantTestsMap, needAll, GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), HandleUpdate);
return(BuildTestRunResult(testResults, expectedTests));
}
