public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults();
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
}
private void ReportTestStart(string line)
{
string qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim();
TestCase testCase = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun);
_reporter.ReportTestsStarted(testCase.Yield());
}
private void CheckStandardOutputResultParser(IEnumerable <TestCase> testCasesRun, IEnumerable <string> consoleOutput,
IList <TestResult> results, TestCase crashedTestCase)
{
var parser = new StandardOutputTestResultParser(testCasesRun, consoleOutput, MockLogger.Object);
parser.GetTestResults().Should().BeEquivalentTo(results);
parser.CrashedTestCase.Should().Be(crashedTestCase);
}
private List <TestResult> ComputeTestResults(List <string> consoleOutput)
{
IList <TestCase> cases = new List <TestCase>();
cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\");
return(parser.GetTestResults());
}
public void ReportLine(string line)
{
if (StandardOutputTestResultParser.IsRunLine(line))
{
if (_consoleOutput.Count > 0)
{
ReportTestResult();
}
ReportTestStart(line);
}
_consoleOutput.Add(line);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithPrefixingTest, TestEnvironment.Logger)
.GetTestResults();
results.Count.Should().Be(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
private TimeSpan ParseDuration(string durationInSeconds)
{
return
(StandardOutputTestResultParser
.NormalizeDuration(TimeSpan.FromSeconds(double.Parse(durationInSeconds, NumberFormatInfo))));
}
private List<TestResult> ComputeTestResults(List<string> consoleOutput)
{
IList<TestCase> cases = new List<TestCase>();
cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\");
return parser.GetTestResults();
}
private TestResult CreateTestResult()
{
int currentLineIndex = 0;
while (currentLineIndex < _consoleOutput.Count &&
!StandardOutputTestResultParser.IsRunLine(_consoleOutput[currentLineIndex]))
{
currentLineIndex++;
}
if (currentLineIndex >= _consoleOutput.Count)
{
return(null);
}
string line = _consoleOutput[currentLineIndex++];
string qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim();
TestCase testCase = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun);
if (currentLineIndex >= _consoleOutput.Count)
{
CrashedTestCase = testCase;
return(StandardOutputTestResultParser.CreateFailedTestResult(
testCase,
TimeSpan.FromMilliseconds(0),
StandardOutputTestResultParser.CrashText,
""));
}
line = _consoleOutput[currentLineIndex++];
string errorMsg = "";
while (
!(StandardOutputTestResultParser.IsFailedLine(line) ||
StandardOutputTestResultParser.IsPassedLine(line)) &&
currentLineIndex <= _consoleOutput.Count)
{
errorMsg += line + "\n";
line = currentLineIndex < _consoleOutput.Count ? _consoleOutput[currentLineIndex] : "";
currentLineIndex++;
}
if (StandardOutputTestResultParser.IsFailedLine(line))
{
ErrorMessageParser parser = new ErrorMessageParser(errorMsg, _baseDir);
parser.Parse();
return(StandardOutputTestResultParser.CreateFailedTestResult(
testCase,
StandardOutputTestResultParser.ParseDuration(line, _logger),
parser.ErrorMessage,
parser.ErrorStackTrace));
}
if (StandardOutputTestResultParser.IsPassedLine(line))
{
return(StandardOutputTestResultParser.CreatePassedTestResult(
testCase,
StandardOutputTestResultParser.ParseDuration(line, _logger)));
}
CrashedTestCase = testCase;
string message = StandardOutputTestResultParser.CrashText;
message += errorMsg == "" ? "" : $"\nTest output:\n\n{errorMsg}";
TestResult result = StandardOutputTestResultParser.CreateFailedTestResult(
testCase,
TimeSpan.FromMilliseconds(0),
message,
"");
return(result);
}
private List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string resultXmlFile, List<string> consoleOutput, string baseDir)
{
var testResults = new List<TestResult>();
TestCase[] testCasesRunAsArray = testCasesRun as TestCase[] ?? testCasesRun.ToArray();
var xmlParser = new XmlTestResultParser(testCasesRunAsArray, resultXmlFile, _testEnvironment, baseDir);
var consoleParser = new StandardOutputTestResultParser(testCasesRunAsArray, consoleOutput, _testEnvironment, baseDir);
testResults.AddRange(xmlParser.GetTestResults());
_testEnvironment.DebugInfo($"Collected {testResults.Count} test results from XML result file '{resultXmlFile}'");
if (testResults.Count < testCasesRunAsArray.Length)
{
List<TestResult> consoleResults = consoleParser.GetTestResults();
int nrOfCollectedTestResults = 0;
// ReSharper disable once AccessToModifiedClosure
foreach (TestResult testResult in consoleResults.Where(tr => !testResults.Exists(tr2 => tr.TestCase.FullyQualifiedName == tr2.TestCase.FullyQualifiedName)))
{
testResults.Add(testResult);
nrOfCollectedTestResults++;
}
_testEnvironment.DebugInfo($"Collected {nrOfCollectedTestResults} test results from console output");
}
if (testResults.Count < testCasesRunAsArray.Length)
{
string errorMessage, errorStackTrace = null;
if (consoleParser.CrashedTestCase == null)
{
errorMessage = "";
}
else
{
errorMessage = $"reason is probably a crash of test {consoleParser.CrashedTestCase.DisplayName}";
errorStackTrace = ErrorMessageParser.CreateStackTraceEntry("crash suspect",
consoleParser.CrashedTestCase.CodeFilePath, consoleParser.CrashedTestCase.LineNumber.ToString());
}
int nrOfCreatedTestResults = 0;
// ReSharper disable once AccessToModifiedClosure
foreach (TestCase testCase in testCasesRunAsArray.Where(tc => !testResults.Exists(tr => tr.TestCase.FullyQualifiedName == tc.FullyQualifiedName)))
{
testResults.Add(new TestResult(testCase)
{
ComputerName = Environment.MachineName,
Outcome = TestOutcome.Skipped,
ErrorMessage = errorMessage,
ErrorStackTrace = errorStackTrace
});
nrOfCreatedTestResults++;
}
_testEnvironment.DebugInfo($"Created {nrOfCreatedTestResults} test results for tests which were neither found in result XML file nor in console output");
}
testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
return testResults;
}
