public void GetCommandLines_AllTests_ProducesCorrectArguments()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}");
}
public void GetTestResults_Sample1_UnexpectedTestOutcome_LogsErrorAndThrows()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_007");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
parser.Invoking(p => p.GetTestResults()).Should().NotThrow <Exception>();
MockLogger.Verify(l => l.LogError(It.Is <string>(s => s.Contains("Foo"))), Times.Exactly(1));
}
public void GetCommandLines_ShuffleTestsWithDefaultSeed_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.ShuffleTests).Returns(true);
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{GoogleTestConstants.ShuffleTestsOption}");
}
public void GetTestResults_Sample2_FindsPassedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("FooTest.DoesXyz");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile2, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7 # GetParam() = (false, 200, 0)");
var parser = new XmlTestResultParser(testCases, TestResources.XmlFile1, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(1);
AssertTestResultIsPassed(results[0]);
}
public void GetTestResults_Sample1_FindsPassedParameterizedResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("ParameterizedTestsTest1/AllEnabledTest.TestInstance/7");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
}
public void Filter_ExpressionMatchesNothing_EmptyResult()
{
_mockFilterExpression.Setup(e => e.MatchTestCase(It.IsAny <TestCase>(), It.IsAny <Func <string, object> >())).Returns(false);
IEnumerable <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar", "Foo.Baz").Select(tc => tc.ToVsTestCase());
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
IEnumerable <TestCase> filteredTestCases = filter.Filter(testCases).ToList();
AssertAreEqual(new List <TestCase>(), filteredTestCases);
}
public void Filter_ExpressionAcceptsAnything_NoFiltering()
{
_mockFilterExpression.Setup(e => e.MatchTestCase(It.IsAny <TestCase>(), It.IsAny <Func <string, object> >())).Returns(true);
IEnumerable <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar", "Foo.Baz").Select(DataConversionExtensions.ToVsTestCase);
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
IEnumerable <TestCase> filteredTestCases = filter.Filter(testCases).ToList();
AssertAreEqual(testCases, filteredTestCases);
}
public void GetCommandLines_TestsWithCommonSuite_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.FooTest"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:");
}
public void GetTestResults_Umlauts_FindsOneResultAndWarns()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("TheClass.Täst1", "TheClass.Töst1", "TheClass.Täst2");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlUmlauts, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
AssertTestResultIsPassed(results[0]);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Once);
}
public void GetTestResults_FileDoesNotExist_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest",
"FooSuite.BazTest", "BarSuite.BazTest2");
var parser = new XmlTestResultParser(testCases, "someexecutable", "somefile", TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().BeEmpty();
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_Sample1_FindsPassedAndSkipptedResults()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001", "SimpleTest.DISABLED_TestMethodDisabled");
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFile1, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().HaveCount(2);
AssertTestResultIsPassed(results[0]);
AssertTestResultIsSkipped(results[1]);
}
public void GetCommandLines_RepetitionsOption_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.NrOfTestRepetitions).Returns(4711);
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
string repetitionsOption = GoogleTestConstants.NrOfRepetitionsOption + "=4711";
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{repetitionsOption}");
}
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined()
{
IEnumerable <Model.TestCase> testCasesWithDifferentSuite = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest");
IEnumerable <Model.TestCase> allTestCases = testCasesWithDifferentSuite.Union(TestDataCreator.CreateDummyTestCases("FooSuite.BazTest", "BarSuite.BazTest2"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithDifferentSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest");
}
public void Matches_ExpressionMatchesDisplayName_CorrectFiltering()
{
List <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar").Select(tc => tc.ToVsTestCase()).ToList();
_mockFilterExpression.Setup(e => e.MatchTestCase(It.Is <TestCase>(tc => tc == testCases[0]), It.Is <Func <string, object> >(f => f("DisplayName").ToString() == "Foo.Bar"))).Returns(true);
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
bool matches = filter.Matches(testCases[0]);
matches.Should().BeTrue();
}
public void SetPropertyValue_Trait_CorrectValidation()
{
var testCase = TestDataCreator.CreateDummyTestCases("Foo.Bar").Single().ToVsTestCase();
testCase.Traits.Add(new Trait("MyTrait", "value1"));
//registers TestProperty objects for trait names
// ReSharper disable once ObjectCreationAsStatement
new TestCaseFilter(MockRunContext.Object, new HashSet <string> {
"MyTrait"
}, TestEnvironment.Logger);
TestProperty property = TestProperty.Find("MyTrait");
Action action = () => testCase.SetPropertyValue(property, "i");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "_i");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "äöüÄÖÜß$");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "_äöüÄÖÜß$");
action.ShouldNotThrow();
// since we are not at the beginning of the method name
action = () => testCase.SetPropertyValue(property, "1");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "_1");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "_");
action.ShouldNotThrow();
action = () => testCase.SetPropertyValue(property, "");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
action = () => testCase.SetPropertyValue(property, "_(");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
action = () => testCase.SetPropertyValue(property, "a(");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
action = () => testCase.SetPropertyValue(property, "1(");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
action = () => testCase.SetPropertyValue(property, "%");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
action = () => testCase.SetPropertyValue(property, "+");
action.ShouldThrow <ArgumentException>().WithMessage("MyTrait");
}
public void Filter_ExpressionMatchesDisplayName_CorrectFiltering()
{
List <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar", "Foo.Baz").Select(tc => tc.ToVsTestCase()).ToList();
_mockFilterExpression.Setup(e => e.MatchTestCase(It.Is <TestCase>(tc => tc == testCases[0]), It.Is <Func <string, object> >(f => f("DisplayName").ToString() == "Foo.Bar"))).Returns(true);
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
IEnumerable <TestCase> filteredTestCases = filter.Filter(testCases).ToList();
AssertAreEqual(testCases[0].Yield(), filteredTestCases);
}
public void Filter_NoFilterExpressionProvided_NoFiltering()
{
MockRunContext.Setup(rc => rc.GetTestCaseFilter(
It.IsAny <IEnumerable <string> >(), It.IsAny <Func <string, TestProperty> >()))
.Returns((ITestCaseFilterExpression)null);
IEnumerable <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar", "Foo.Baz").Select(tc => tc.ToVsTestCase());
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
IEnumerable <TestCase> filteredTestCases = filter.Filter(testCases).ToList();
AssertAreEqual(testCases, filteredTestCases);
}
public void SplitTestcases_MoreThreadsThanTests_TestsAreDistributedEvenly()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest");
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(8);
ITestsSplitter splitter = new NumberBasedTestsSplitter(testCasesWithCommonSuite, TestEnvironment.Options);
List <List <Model.TestCase> > result = splitter.SplitTestcases();
result.Should().HaveCount(2);
result[0].Should().ContainSingle();
result[1].Should().ContainSingle();
}
public void GetTestResults_InvalidFile_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.DebugMode).Returns(true);
var parser = new XmlTestResultParser(testCases, TestResources.XmlFileBroken, TestEnvironment, "");
List <Model.TestResult> results = parser.GetTestResults();
results.Count.Should().Be(0);
MockLogger.Verify(l => l.LogWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetTestResults_FileWithInvalidStatusAttribute_WarningAndEmptyResult()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("GoogleTestSuiteName1.TestMethod_001",
"GoogleTestSuiteName1.TestMethod_002");
MockOptions.Setup(o => o.OutputMode).Returns(OutputMode.Verbose);
var parser = new XmlTestResultParser(testCases, "someexecutable", TestResources.XmlFileBroken_InvalidStatusAttibute, TestEnvironment.Logger);
List <Model.TestResult> results = parser.GetTestResults();
results.Should().ContainSingle();
MockLogger.Verify(l => l.DebugWarning(It.IsAny <string>()), Times.Exactly(1));
}
public void GetCommandLines_BreakOnFailureOption_IsAppendedCorrectly()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
string breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(false);
commandLine.Should().Contain(breakOnFailureOption);
MockOptions.Setup(o => o.BreakOnFailure).Returns(true);
commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(true);
commandLine.Should().Contain(breakOnFailureOption);
}
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases(
"InstantiationName2/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/1");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be(
$"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0");
}
public void Filter_Trait_CorrectFiltering()
{
List <TestCase> testCases = TestDataCreator.CreateDummyTestCases("Foo.Bar", "Foo.Baz").Select(tc => tc.ToVsTestCase()).ToList();
testCases[0].Traits.Add(new Trait("MyTrait", "value1"));
SetupFilterToAcceptTraitForTestCase(testCases[0], "MyTrait", "value1");
testCases[1].Traits.Add(new Trait("MyTrait", "value2"));
SetupFilterToAcceptTraitForTestCase(testCases[1], "MyTrait", "value2");
_traitNames.Add("MyTrait");
TestCaseFilter filter = new TestCaseFilter(MockRunContext.Object, _traitNames, TestEnvironment.Logger);
IEnumerable <TestCase> filteredTestCases = filter.Filter(testCases).ToList();
AssertAreEqual(testCases, filteredTestCases);
}
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest",
"FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.BarTest"));
IEnumerable <Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse();
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string commandLineFromBackwards = new CommandLineGenerator(allTestCases, testCasesReversed, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string expectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:";
commandLine.Should().Be(expectedCommandLine);
commandLineFromBackwards.Should().Be(expectedCommandLine);
}