public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
parser.Invoking(p => p.GetTestResults()).ShouldNotThrow<Exception>();
MockLogger.Verify(l => l.LogError(It.Is<string>(s => s.Contains("Foo"))), Times.Exactly(1));
}
public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
parser.Invoking(p => p.GetTestResults()).Should().NotThrow <Exception>();
MockLogger.Verify(l => l.LogError(It.Is <string>(s => s.Contains("Foo"))), Times.Exactly(1));
}
public void GetTestResults_Sample2_FindsPassedResult()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("FooTest.DoesXyz");
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile2, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(1, results.Count);
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7 # GetParam() = (false, 200, 0)");
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(1, results.Count);
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7 # GetParam() = (false, 200, 0)");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample2_FindsPassedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("FooTest.DoesXyz");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest",
"FooSuite.BazTest", "BarSuite.BazTest2");
XmlTestResultParser parser = new XmlTestResultParser(testCases, "somefile", TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(0, results.Count);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_Sample1_FindsPassedAndSkipptedResults()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled");
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(2, results.Count);
AssertTestResultIsPassed(results[0]);
AssertTestResultIsSkipped(results[1]);
}
public void GetTestResults_Sample1_FindsPassedAndSkipptedResults()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().HaveCount(2);
AssertTestResultIsPassed(results[0]);
AssertTestResultIsSkipped(results[1]);
}
public void GetTestResults_Sample1_FindsPassedAndSkipptedResults()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(2);
AssertTestResultIsPassed(results[0]);
AssertTestResultIsSkipped(results[1]);
}
public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest",
"FooSuite.BazTest", "BarSuite.BazTest2");
var parser = new XmlTestResultParser(testCases, "someexecutable", "somefile", TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().BeEmpty();
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_Umlauts_FindsOneResultAndWarns()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("TheClass.Täst1", "TheClass.Töst1", "TheClass.Täst2");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlUmlauts, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Once);
}
public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest",
"FooSuite.BazTest", "BarSuite.BazTest2");
var parser = new XmlTestResultParser(testCases, "somefile", TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(0);
MockLogger.Verify(l => l.LogWarning(It.IsAny<string>()), Times.Exactly(1));
}
public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.DebugMode).Returns(true);
var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
MockLogger.Verify(l => l.LogWarning(It.IsAny<string>()), Times.Exactly(1));
}
public void GetTestResults_Sample1_FindsParameterizedFailureResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.ToTestCase("ParameterizedTestsTest1/AllEnabledTest.TestInstance/11", TestDataCreator.DummyExecutable, @"someSimpleParameterizedTest.cpp").Yield();
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
string errorMsg = @"Expected: (0) != ((pGSD->g_outputs64[(g_nOutput[ 8 ]-1)/64] & g_dnOutput[g_nOutput[ 8 ]])), actual: 0 vs 0";
AssertTestResultIsFailure(results[0], errorMsg);
results[0].ErrorStackTrace.Should().Contain(@"someSimpleParameterizedTest.cpp");
}
public void GetTestResults_InvalidFile_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.DebugMode).Returns(true);
var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(0);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.DebugMode).Returns(true);
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFileBroken_InvalidStatusAttibute, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(0, results.Count);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.OutputMode).Returns(OutputMode.Verbose);
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
MockLogger.Verify(l => l.DebugWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_Sample1_FindsParamterizedFailureResult()
{
IEnumerable <Model.TestCase> testCases = ToTestCase("ParameterizedTestsTest1/AllEnabledTest.TestInstance/11 # GetParam() = (true, 0, 100)",
DummyExecutable, @"someSimpleParameterizedTest.cpp").Yield();
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(1, results.Count);
string errorMsg = @"Expected: (0) != ((pGSD->g_outputs64[(g_nOutput[ 8 ]-1)/64] & g_dnOutput[g_nOutput[ 8 ]])), actual: 0 vs 0";
AssertTestResultIsFailure(results[0], errorMsg);
Assert.IsTrue(results[0].ErrorStackTrace.Contains(@"someSimpleParameterizedTest.cpp"));
}
public void GetTestResults_Sample1_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.ToTestCase("AnimalsTest.testGetEnoughAnimals", TestDataCreator.DummyExecutable, @"x:\prod\company\util\util.cpp").Yield();
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
string ErrorMsg = @"Value of: animals.size()
Actual: 1
Expected: 3
Should get three animals";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"x:\prod\company\util\util.cpp");
}
public void GetTestResults_Sample1_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = ToTestCase("AnimalsTest.testGetEnoughAnimals",
DummyExecutable, @"x:\prod\company\util\util.cpp").Yield();
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(1, results.Count);
string ErrorMsg = @"Value of: animals.size()
Actual: 1
Expected: 3
Should get three animals";
AssertTestResultIsFailure(results[0], ErrorMsg);
Assert.IsTrue(results[0].ErrorStackTrace.Contains(@"x:\prod\company\util\util.cpp"));
}
public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007");
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile1, TestEnvironment, "");
try
{
parser.GetTestResults();
Assert.Fail();
}
catch (Exception)
{
// that's expected
}
MockLogger.Verify(l => l.LogError(It.Is <string>(s => s.Contains("Foo"))), Times.Exactly(1));
}
public void GetTestResults_Sample2_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.ToTestCase("FooTest.MethodBarDoesAbc", TestDataCreator.DummyExecutable,
@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield();
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
string ErrorMsg = @"#1 - Value of: output_filepath
Actual: ""this/package/testdata/myoutputfile.dat""
Expected: input_filepath
Which is: ""this/package/testdata/myinputfile.dat""
Something's not right!!
#2 - Value of: 56456
Expected: 12312
Something's wrong :(";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp");
}
public void GetTestResults_Sample2_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = ToTestCase("FooTest.MethodBarDoesAbc", DummyExecutable,
@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield();
XmlTestResultParser parser = new XmlTestResultParser(testCases, XmlFile2, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
Assert.AreEqual(1, results.Count);
string ErrorMsg = @"#1 - Value of: output_filepath
Actual: ""this/package/testdata/myoutputfile.dat""
Expected: input_filepath
Which is: ""this/package/testdata/myinputfile.dat""
Something's not right!!
#2 - Value of: 56456
Expected: 12312
Something's wrong :(";
AssertTestResultIsFailure(results[0], ErrorMsg);
Assert.IsTrue(results[0].ErrorStackTrace.Contains(@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp"));
}
public void GetTestResults_Sample1_FindsFailureResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("AnimalsTest.testGetEnoughAnimals", TestDataCreator.DummyExecutable, @"x:\prod\company\util\util.cpp").Yield();
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
string ErrorMsg = @"Value of: animals.size()
Actual: 1
Expected: 3
Should get three animals";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"x:\prod\company\util\util.cpp");
}
public void GetTestResults_Sample1_FindsParamterizedFailureResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("ParameterizedTestsTest1/AllEnabledTest.TestInstance/11", TestDataCreator.DummyExecutable, @"someSimpleParameterizedTest.cpp").Yield();
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
string errorMsg = @"Expected: (0) != ((pGSD->g_outputs64[(g_nOutput[ 8 ]-1)/64] & g_dnOutput[g_nOutput[ 8 ]])), actual: 0 vs 0";
AssertTestResultIsFailure(results[0], errorMsg);
results[0].ErrorStackTrace.Should().Contain(@"someSimpleParameterizedTest.cpp");
}
public void GetTestResults_Sample2_FindsPassedResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("FooTest.DoesXyz");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample2_FindsFailureResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.ToTestCase("FooTest.MethodBarDoesAbc", TestDataCreator.DummyExecutable,
@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield();
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile2, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
string ErrorMsg = @"#1 - Value of: output_filepath
Actual: ""this/package/testdata/myoutputfile.dat""
Expected: input_filepath
Which is: ""this/package/testdata/myinputfile.dat""
Something's not right!!
#2 - Value of: 56456
Expected: 12312
Something's wrong :(";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp");
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List<Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
AssertTestResultIsPassed(results[0]);
}
private List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string resultXmlFile, List<string> consoleOutput, string baseDir)
{
var testResults = new List<TestResult>();
TestCase[] testCasesRunAsArray = testCasesRun as TestCase[] ?? testCasesRun.ToArray();
var xmlParser = new XmlTestResultParser(testCasesRunAsArray, resultXmlFile, _testEnvironment, baseDir);
var consoleParser = new StandardOutputTestResultParser(testCasesRunAsArray, consoleOutput, _testEnvironment, baseDir);
testResults.AddRange(xmlParser.GetTestResults());
_testEnvironment.DebugInfo($"Collected {testResults.Count} test results from XML result file '{resultXmlFile}'");
if (testResults.Count < testCasesRunAsArray.Length)
{
List<TestResult> consoleResults = consoleParser.GetTestResults();
int nrOfCollectedTestResults = 0;
// ReSharper disable once AccessToModifiedClosure
foreach (TestResult testResult in consoleResults.Where(tr => !testResults.Exists(tr2 => tr.TestCase.FullyQualifiedName == tr2.TestCase.FullyQualifiedName)))
{
testResults.Add(testResult);
nrOfCollectedTestResults++;
}
_testEnvironment.DebugInfo($"Collected {nrOfCollectedTestResults} test results from console output");
}
if (testResults.Count < testCasesRunAsArray.Length)
{
string errorMessage, errorStackTrace = null;
if (consoleParser.CrashedTestCase == null)
{
errorMessage = "";
}
else
{
errorMessage = $"reason is probably a crash of test {consoleParser.CrashedTestCase.DisplayName}";
errorStackTrace = ErrorMessageParser.CreateStackTraceEntry("crash suspect",
consoleParser.CrashedTestCase.CodeFilePath, consoleParser.CrashedTestCase.LineNumber.ToString());
}
int nrOfCreatedTestResults = 0;
// ReSharper disable once AccessToModifiedClosure
foreach (TestCase testCase in testCasesRunAsArray.Where(tc => !testResults.Exists(tr => tr.TestCase.FullyQualifiedName == tc.FullyQualifiedName)))
{
testResults.Add(new TestResult(testCase)
{
ComputerName = Environment.MachineName,
Outcome = TestOutcome.Skipped,
ErrorMessage = errorMessage,
ErrorStackTrace = errorStackTrace
});
nrOfCreatedTestResults++;
}
_testEnvironment.DebugInfo($"Created {nrOfCreatedTestResults} test results for tests which were neither found in result XML file nor in console output");
}
testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
return testResults;
}