private List <TestResult> ComputeTestResults(List <string> consoleOutput)
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment.Logger);
return(parser.GetTestResults());
}
public void GetTestResults_Sample1_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.ToTestCase("AnimalsTest.testGetEnoughAnimals", TestDataCreator.DummyExecutable, @"x:\prod\company\util\util.cpp").Yield();
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
string ErrorMsg = @"Value of: animals.size()
Actual: 1
Expected: 3
Should get three animals";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"x:\prod\company\util\util.cpp");
}
public void SplitTestcases_AsymmetricCase_TestsAreDistributedCorrectly()
{
IDictionary <Model.TestCase, int> durations = new Dictionary <Model.TestCase, int>();
durations.Add(TestDataCreator.ToTestCase("ShortTest1"), 1);
durations.Add(TestDataCreator.ToTestCase("LongTest"), 5);
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(3);
ITestsSplitter splitter = new DurationBasedTestsSplitter(durations, TestEnvironment.Options);
List <List <Model.TestCase> > result = splitter.SplitTestcases();
result.Count.Should().Be(2);
result[0].Count.Should().Be(1);
result[0][0].FullyQualifiedName.Should().Be("LongTest");
result[1].Count.Should().Be(1);
}
private void DoTestBeginOfErrorMessage(bool inVisualStudio, string beginOfErrorMessage)
{
var errorMessage = "My error message";
var result = new TestResult(TestDataCreator.ToTestCase("MyTestCase"))
{
Outcome = TestOutcome.Failed,
ErrorMessage = errorMessage,
ComputerName = "My Computer"
};
var reporter = new VsTestFrameworkReporter(MockFrameworkHandle.Object, inVisualStudio, MockLogger.Object);
reporter.ReportTestResults(result.Yield());
MockFrameworkHandle.Verify(h => h.RecordResult(
It.Is <VsTestResult>(tr => tr.ErrorMessage.Equals(beginOfErrorMessage + errorMessage))),
Times.Exactly(1));
}
public void ReportTestResults_FromVsTestConsole_StacktraceEndsWithoutNewline()
{
var errorStacktrace = "My stack trace";
var result = new TestResult(TestDataCreator.ToTestCase("MyTestCase"))
{
Outcome = TestOutcome.Failed,
ErrorStackTrace = errorStacktrace + Environment.NewLine,
ComputerName = "My Computer"
};
var reporter = new VsTestFrameworkReporter(MockFrameworkHandle.Object, false, MockLogger.Object);
reporter.ReportTestResults(result.Yield());
MockFrameworkHandle.Verify(h => h.RecordResult(
It.Is <VsTestResult>(tr => tr.ErrorStackTrace.Equals(errorStacktrace))),
Times.Exactly(1));
}
private IList <TestResult> ComputeTestResults(List <string> consoleOutput)
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, MockLogger.Object, MockFrameworkReporter.Object);
consoleOutput.ForEach(parser.ReportLine);
parser.Flush();
return(parser.TestResults);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithPrefixingTest, TestEnvironment.Logger)
.GetTestResults();
results.Count.Should().Be(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
public void SplitTestcases_SimpleCase_TestsAreDistributedCorrectly()
{
IDictionary <Model.TestCase, int> durations = new Dictionary <Model.TestCase, int>
{
{ TestDataCreator.ToTestCase("ShortTest1"), 1 },
{ TestDataCreator.ToTestCase("ShortTest2"), 1 },
{ TestDataCreator.ToTestCase("LongTest"), 3 },
{ TestDataCreator.ToTestCase("ShortTest3"), 1 }
};
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(2);
ITestsSplitter splitter = new DurationBasedTestsSplitter(durations, TestEnvironment.Options);
List <List <Model.TestCase> > result = splitter.SplitTestcases();
result.Should().HaveCount(2);
result[0].Should().ContainSingle();
result[0][0].FullyQualifiedName.Should().Be("LongTest");
result[1].Should().HaveCount(3);
}
public void GetTestResults_Sample2_FindsFailureResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.ToTestCase("FooTest.MethodBarDoesAbc", TestDataCreator.DummyExecutable,
@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp").Yield();
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
string ErrorMsg = @"#1 - Value of: output_filepath
Actual: ""this/package/testdata/myoutputfile.dat""
Expected: input_filepath
Which is: ""this/package/testdata/myinputfile.dat""
Something's not right!!
#2 - Value of: 56456
Expected: 12312
Something's wrong :(";
AssertTestResultIsFailure(results[0], ErrorMsg);
results[0].ErrorStackTrace.Should().Contain(@"c:\prod\gtest-1.7.0\staticallylinkedgoogletests\main.cpp");
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
ConsoleOutputWithPrefixingTest.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
}
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
{
string[] consoleOutput =
{
@"[==========] Running 2 tests from 1 test case.",
@"[----------] Global test environment set-up.",
@"[----------] 2 tests from TestMath",
@"[ RUN ] Test.AB",
@"[ OK ] Test.A(0 ms)",
@"[ RUN ] Test.A",
@"[ OK ] Test.A(0 ms)",
@"[----------] 2 tests from TestMath(26 ms total)",
@"",
@"[----------] Global test environment tear-down",
@"[==========] 2 tests from 1 test case ran. (36 ms total)",
@"[ PASSED ] 2 test.",
};
var cases = new List <TestCase>
{
TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
@"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
};
var parser = new StreamingStandardOutputTestResultParser(cases, TestEnvironment.Logger, MockFrameworkReporter.Object);
consoleOutput.ToList().ForEach(parser.ReportLine);
parser.Flush();
var results = parser.TestResults;
results.Should().HaveCount(2);
results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
CheckStandardOutputResultParser(cases, consoleOutput, results, parser.CrashedTestCase);
}
public void SplitTestcases_ALotMoreTestsThanThreads_TestsAreDistributedEvenly()
{
List <Model.TestCase> testcases = new List <Model.TestCase>();
for (int i = 0; i < 5002; i++)
{
testcases.Add(TestDataCreator.ToTestCase("TestSuite.Test" + i));
}
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(8);
ITestsSplitter splitter = new NumberBasedTestsSplitter(testcases, TestEnvironment.Options);
List <List <Model.TestCase> > result = splitter.SplitTestcases();
result.Should().HaveCount(8);
result[0].Should().HaveCount(626);
result[1].Should().HaveCount(626);
result[2].Should().HaveCount(625);
result[3].Should().HaveCount(625);
result[4].Should().HaveCount(625);
result[5].Should().HaveCount(625);
result[6].Should().HaveCount(625);
result[7].Should().HaveCount(625);
}
public void ReadTestDurations_NoDurationFile_EmptyDictionary()
{
string tempFile = Path.GetTempFileName();
var serializer = new TestDurationSerializer();
IDictionary <Model.TestCase, int> durations = serializer.ReadTestDurations(TestDataCreator.ToTestCase("TestSuite1.Test1", tempFile).Yield());
durations.Should().NotBeNull();
durations.Should().BeEmpty();
}