private TestResult CreateExitCodeTestResult(TestCase exitCodeTestCase, ExecutableResult executableResult)
{
string message = "";
if (executableResult.ExitCode != 0)
{
message += $"Exit code: {executableResult.ExitCode}";
}
if (executableResult.ExitCodeOutput.Any())
{
if (!string.IsNullOrWhiteSpace(message))
{
message += $"{Environment.NewLine}{Environment.NewLine}";
}
message += string.Join(Environment.NewLine, executableResult.ExitCodeOutput);
}
var result = StreamingStandardOutputTestResultParser
.CreateFailedTestResult(exitCodeTestCase, TimeSpan.Zero, message, "");
result.Outcome = executableResult.ExitCodeSkip
? TestOutcome.Skipped
: executableResult.ExitCode == 0
? TestOutcome.Passed
: TestOutcome.Failed;
return(result);
}
public void GetTestResults_OutputWithSkip_ExitCodeSkipIsTrue()
{
string[] consoleOutput =
{
@"[==========] Running 1 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 1 tests from TestMath",
@"[ RUN ] TestMath.AddPasses",
@"Some output produced by the exe",
@"[ OK ] TestMath.AddPasses(0 ms)",
@"[----------] 1 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 3 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 1 test.",
@"",
StreamingStandardOutputTestResultParser.GtaExitCodeSkip,
StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin,
"Output 1",
"Output 2",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
parser.ExitCodeSkip.Should().BeTrue();
}
public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
ConsoleOutputWithSkippedTestAsLastTest.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
}
public void GetTestResults_OutputWithConsoleOutput_ConsoleOutputIsIgnored()
{
string[] consoleOutput =
{
@"[==========] Running 1 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 1 tests from TestMath",
@"[ RUN ] TestMath.AddPasses",
@"Some output produced by the exe",
@"[ OK ] TestMath.AddPasses(0 ms)",
@"[----------] 1 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 3 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 1 test.",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().ContainSingle();
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithInvalidDurationUnit_DefaultDurationIsUsedAndWarningIsProduced()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 s)"
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().ContainSingle();
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(1));
results[0].ErrorStackTrace.Should().Contain(@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
MockLogger.Verify(l => l.LogWarning(
It.Is <string>(s => s.Contains("'[ FAILED ] TestMath.AddFails (3 s)'"))), Times.Exactly(1));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithPrefixedFailedLine_FailingTestIsRecognized()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses",
@"DummyOutput[ FAILED ] TestMath.AddPasses(0 ms)",
@"[ RUN ] TestMath.Crash",
@"unknown file: error: SEH exception with code 0xc0000005 thrown in the test body.",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(3);
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain("DummyOutput");
results[1].Duration.Should().Be(StreamingStandardOutputTestResultParser.ShortTestDuration);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_CompleteOutput_ParsedCorrectly()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses",
@"[ OK ] TestMath.AddPasses(0 ms)",
@"[ RUN ] TestMath.Crash",
@"unknown file: error: SEH exception with code 0xc0000005 thrown in the test body.",
@"[ FAILED ] TestMath.Crash(9 ms)",
@"[----------] 3 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 3 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 1 test.",
@"[ FAILED ] 2 tests, listed below:",
@"[ FAILED ] TestMath.AddFails",
@"[ FAILED ] TestMath.Crash",
@"",
@" 2 FAILED TESTS",
@"",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(3);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should()
.Contain(
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
results[1].Duration.Should().Be(StreamingStandardOutputTestResultParser.ShortTestDuration);
results[2].TestCase.FullyQualifiedName.Should().Be("TestMath.Crash");
XmlTestResultParserTests.AssertTestResultIsFailure(results[2]);
results[2].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[2].Duration.Should().Be(TimeSpan.FromMilliseconds(9));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
string[] consoleOutput = @"[==========] Running 3 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 3 tests from Test
[ RUN ] Test.Succeed
[ OK ] Test.Succeed (0 ms)
[ RUN ] Test.Fail
C:\...\test.cpp(14): error: Value of: false
Actual: false
Expected: true
[ FAILED ] Test.Fail (0 ms)
[ RUN ] Test.Skip
[ SKIPPED ] Test.Skip (1 ms)
[----------] 3 tests from Test (3 ms total)
[----------] Global test environment tear-down
[==========] 3 tests from 1 test suite ran. (6 ms total)
[ PASSED ] 1 test.
[ SKIPPED ] 1 test, listed below:
[ SKIPPED ] Test.Skip
[ FAILED ] 1 test, listed below:
[ FAILED ] Test.Fail
1 FAILED TEST
".Split('\n');
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public IList <TestResult> GetTestResults()
{
var streamingParser = new StreamingStandardOutputTestResultParser(_testCasesRun, _logger, new DummyTestFrameworkReporter());
_consoleOutput.ForEach(l => streamingParser.ReportLine(l));
streamingParser.Flush();
CrashedTestCase = streamingParser.CrashedTestCase;
return(streamingParser.TestResults);
}
private IList <TestResult> GetTestResultsFromCompleteOutputFile()
{
var testCases = new GoogleTestDiscoverer(MockLogger.Object, MockOptions.Object, new DefaultDiaResolverFactory())
.GetTestsFromExecutable(TestResources.Tests_ReleaseX64);
var parser = new StreamingStandardOutputTestResultParser(testCases, MockLogger.Object, MockFrameworkReporter.Object);
CompleteStandardOutput.ForEach(parser.ReportLine);
parser.Flush();
return(parser.TestResults);
}
private IList <TestResult> ComputeTestResults(List <string> consoleOutput)
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ForEach(parser.ReportLine);
parser.Flush();
return(parser.TestResults);
}
public void OutputHandling_OneLine_IsParsedCorrectly()
{
var consoleOutput = File.ReadAllLines(TestResources.Tests_ReleaseX64_Output, Encoding.Default);
var testCases = new GoogleTestDiscoverer(MockLogger.Object, MockOptions.Object)
.GetTestsFromExecutable(TestResources.Tests_ReleaseX64);
var parser = new StreamingStandardOutputTestResultParser(testCases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
var testResult = results.Single(tr => tr.DisplayName == "OutputHandling.OneLine");
var expectedErrorMessage =
"before test\nExpected: 1\nTo be equal to: 2\nafter test";
testResult.ErrorMessage.Should().Be(expectedErrorMessage);
CheckStandardOutputResultParser(testCases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
string[] consoleOutput =
{
@"[==========] Running 2 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 2 tests from TestMath",
@"[ RUN ] Test.AB",
@"[ OK ] Test.A(0 ms)",
@"[ RUN ] Test.A",
@"[ OK ] Test.A(0 ms)",
@"[----------] 2 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 2 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 2 test.",
};
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
ConsoleOutputWithPrefixingTest.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses"
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should().Contain(@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain(StreamingStandardOutputTestResultParser.CrashText);
results[1].ErrorMessage.Should().NotContain("Test output:");
results[1].Duration.Should().Be(TimeSpan.FromMilliseconds(0));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithThousandsSeparatorInDuration_ParsedCorrectly()
{
CultureInfo currentCulture = Thread.CurrentThread.CurrentCulture;
Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US");
try
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (4,656 ms)",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().ContainSingle();
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(4656));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
finally
{
Thread.CurrentThread.CurrentCulture = currentCulture;
}
}
private TimeSpan ParseDuration(string durationInSeconds)
{
return
(StreamingStandardOutputTestResultParser
.NormalizeDuration(TimeSpan.FromSeconds(double.Parse(durationInSeconds, NumberFormatInfo))));
}