public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
ConsoleOutputWithSkippedTestAsLastTest.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
}
public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults();
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
}
public void GetTestResults_OutputWithConsoleOutput_ConsoleOutputIsIgnored()
{
string[] consoleOutput =
{
@"[==========] Running 1 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 1 tests from TestMath",
@"[ RUN ] TestMath.AddPasses",
@"Some output produced by the exe",
@"[ OK ] TestMath.AddPasses(0 ms)",
@"[----------] 1 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 3 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 1 test.",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().ContainSingle();
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithPrefixedFailedLine_FailingTestIsRecognized()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses",
@"DummyOutput[ FAILED ] TestMath.AddPasses(0 ms)",
@"[ RUN ] TestMath.Crash",
@"unknown file: error: SEH exception with code 0xc0000005 thrown in the test body.",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(3);
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain("DummyOutput");
results[1].Duration.Should().Be(StreamingStandardOutputTestResultParser.ShortTestDuration);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithConsoleOutput_ConsoleOutputIsIgnored()
{
IList <TestResult> results = ComputeTestResults(PassingTestProducesConsoleOutput);
results.Count.Should().Be(1);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_CompleteOutput_ParsedCorrectly()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses",
@"[ OK ] TestMath.AddPasses(0 ms)",
@"[ RUN ] TestMath.Crash",
@"unknown file: error: SEH exception with code 0xc0000005 thrown in the test body.",
@"[ FAILED ] TestMath.Crash(9 ms)",
@"[----------] 3 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 3 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 1 test.",
@"[ FAILED ] 2 tests, listed below:",
@"[ FAILED ] TestMath.AddFails",
@"[ FAILED ] TestMath.Crash",
@"",
@" 2 FAILED TESTS",
@"",
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(3);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should()
.Contain(
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
results[1].Duration.Should().Be(StreamingStandardOutputTestResultParser.ShortTestDuration);
results[2].TestCase.FullyQualifiedName.Should().Be("TestMath.Crash");
XmlTestResultParserTests.AssertTestResultIsFailure(results[2]);
results[2].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[2].Duration.Should().Be(TimeSpan.FromMilliseconds(9));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
{
string[] consoleOutput = @"[==========] Running 3 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 3 tests from Test
[ RUN ] Test.Succeed
[ OK ] Test.Succeed (0 ms)
[ RUN ] Test.Fail
C:\...\test.cpp(14): error: Value of: false
Actual: false
Expected: true
[ FAILED ] Test.Fail (0 ms)
[ RUN ] Test.Skip
[ SKIPPED ] Test.Skip (1 ms)
[----------] 3 tests from Test (3 ms total)
[----------] Global test environment tear-down
[==========] 3 tests from 1 test suite ran. (6 ms total)
[ PASSED ] 1 test.
[ SKIPPED ] 1 test, listed below:
[ SKIPPED ] Test.Skip
[ FAILED ] 1 test, listed below:
[ FAILED ] Test.Fail
1 FAILED TEST
".Split('\n');
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(3);
var result = results[0];
result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
XmlTestResultParserTests.AssertTestResultIsPassed(result);
result = results[1];
result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
XmlTestResultParserTests.AssertTestResultIsFailure(result);
result = results[2];
result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
XmlTestResultParserTests.AssertTestResultIsSkipped(result);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithPrefixedPassedLine_PassingTestIsRecognized()
{
IList <TestResult> results = ComputeTestResults(WithPrefixingOutputPassing);
results.Count.Should().Be(3);
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
results[1].Duration.Should().Be(StandardOutputTestResultParser.ShortTestDuration);
}
public void GetTestResults_OutputWithPrefixedFailedLine_FailingTestIsRecognized()
{
IList <TestResult> results = ComputeTestResults(WithPrefixingOutputFailing);
results.Count.Should().Be(3);
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain("DummyOutput");
results[1].Duration.Should().Be(StandardOutputTestResultParser.ShortTestDuration);
}
public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText()
{
IList <TestResult> results = ComputeTestResults(CrashesImmediately);
results.Count.Should().Be(2);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should().Contain(@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain(StandardOutputTestResultParser.CrashText);
results[1].ErrorMessage.Should().NotContain("Test output:");
results[1].Duration.Should().Be(TimeSpan.FromMilliseconds(0));
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithPrefixingTest, TestEnvironment.Logger)
.GetTestResults();
results.Count.Should().Be(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
ConsoleOutputWithPrefixingTest.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
string[] consoleOutput =
{
@"[==========] Running 2 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 2 tests from TestMath",
@"[ RUN ] Test.AB",
@"[ OK ] Test.A(0 ms)",
@"[ RUN ] Test.A",
@"[ OK ] Test.A(0 ms)",
@"[----------] 2 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 2 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 2 test.",
};
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithImmediateCrash_CorrectResultHasCrashText()
{
string[] consoleOutput =
{
@"[==========] Running 3 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 3 tests from TestMath",
@"[ RUN ] TestMath.AddFails",
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp(6): error: Value of: Add(10, 10)",
@" Actual: 20",
@"Expected: 1000",
@"[ FAILED ] TestMath.AddFails (3 ms)",
@"[ RUN ] TestMath.AddPasses"
};
var cases = GetTestCases();
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
IList <TestResult> results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StreamingStandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should().Contain(@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsFailure(results[1]);
results[1].ErrorMessage.Should().Contain(StreamingStandardOutputTestResultParser.CrashText);
results[1].ErrorMessage.Should().NotContain("Test output:");
results[1].Duration.Should().Be(TimeSpan.FromMilliseconds(0));
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void GetTestResults_OutputWithCrashAfterErrorMessage_CorrectResultHasCrashText()
{
IList <TestResult> results = ComputeTestResults(CrashesAfterErrorMsg);
results.Count.Should().Be(3);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should().Contain(@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
results[1].Duration.Should().Be(StandardOutputTestResultParser.ShortTestDuration);
results[2].TestCase.FullyQualifiedName.Should().Be("TestMath.Crash");
XmlTestResultParserTests.AssertTestResultIsFailure(results[2]);
results[2].ErrorMessage.Should().Contain(StandardOutputTestResultParser.CrashText);
results[2].ErrorMessage.Should().Contain("Test output:");
results[2].ErrorMessage.Should().Contain("unknown file: error: SEH exception with code 0xc0000005 thrown in the test body.");
results[2].Duration.Should().Be(TimeSpan.FromMilliseconds(0));
}
public void GetTestResults_CompleteOutput_ParsedCorrectly()
{
IList <TestResult> results = ComputeTestResults(Complete);
results.Count.Should().Be(3);
results[0].TestCase.FullyQualifiedName.Should().Be("TestMath.AddFails");
XmlTestResultParserTests.AssertTestResultIsFailure(results[0]);
results[0].ErrorMessage.Should().NotContain(StandardOutputTestResultParser.CrashText);
results[0].Duration.Should().Be(TimeSpan.FromMilliseconds(3));
results[0].ErrorStackTrace.Should()
.Contain(
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp");
results[1].TestCase.FullyQualifiedName.Should().Be("TestMath.AddPasses");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
results[1].Duration.Should().Be(StandardOutputTestResultParser.ShortTestDuration);
results[2].TestCase.FullyQualifiedName.Should().Be("TestMath.Crash");
XmlTestResultParserTests.AssertTestResultIsFailure(results[2]);
results[2].ErrorMessage.Should().NotContain(StandardOutputTestResultParser.CrashText);
results[2].Duration.Should().Be(TimeSpan.FromMilliseconds(9));
}
