// ReSharper disable once UnusedParameter.Local private void RunTestsFromExecutable(string executable, IEnumerable <TestCase> allTestCases, IEnumerable <TestCase> testCasesToRun, string baseDir, string userParameters, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher) { string resultXmlFile = Path.GetTempFileName(); string workingDir = Path.GetDirectoryName(executable); var serializer = new TestDurationSerializer(); var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, TestEnvironment); foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines()) { if (Canceled) { break; } FrameworkReporter.ReportTestsStarted(arguments.TestCases); TestEnvironment.DebugInfo("Executing command '" + executable + " " + arguments.CommandLine + "'."); List <string> consoleOutput = new TestProcessLauncher(TestEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, TestEnvironment.Options.PrintTestOutput && !TestEnvironment.Options.ParallelTestExecution, false, debuggedLauncher); IEnumerable <TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir); FrameworkReporter.ReportTestResults(results); serializer.UpdateTestDurations(results); } }
// ReSharper disable once UnusedParameter.Local private void RunTestsFromExecutable(string executable, string workingDir, IEnumerable<TestCase> allTestCases, IEnumerable<TestCase> testCasesToRun, string baseDir, string userParameters, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher) { string resultXmlFile = Path.GetTempFileName(); var serializer = new TestDurationSerializer(); var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, _testEnvironment); foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines()) { if (_canceled) { break; } _frameworkReporter.ReportTestsStarted(arguments.TestCases); List<string> consoleOutput = new TestProcessLauncher(_testEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, _testEnvironment.Options.PrintTestOutput && !_testEnvironment.Options.ParallelTestExecution, false, debuggedLauncher); IEnumerable<TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir); Stopwatch stopwatch = Stopwatch.StartNew(); _frameworkReporter.ReportTestResults(results); stopwatch.Stop(); _testEnvironment.DebugInfo($"Reported {results.Count()} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}"); serializer.UpdateTestDurations(results); } }
public void GetCommandLines_AllTests_ProducesCorrectArguments() { IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}"); }
public void GetCommandLines_AllTests_ProducesCorrectArguments() { IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; Assert.AreEqual("--gtest_output=\"xml:\"" + DefaultArgs, commandLine); }
public void GetCommandLines_AllTests_ProducesCorrectArguments() { IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}"); }
// ReSharper disable once UnusedParameter.Local private void RunTestsFromExecutable(string executable, string workingDir, IEnumerable <TestCase> allTestCases, IEnumerable <TestCase> testCasesToRun, string baseDir, string userParameters, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher) { string resultXmlFile = Path.GetTempFileName(); var serializer = new TestDurationSerializer(); var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, _testEnvironment); foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines()) { if (_canceled) { break; } _frameworkReporter.ReportTestsStarted(arguments.TestCases); List <string> consoleOutput = new TestProcessLauncher(_testEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, _testEnvironment.Options.PrintTestOutput && !_testEnvironment.Options.ParallelTestExecution, false, debuggedLauncher); IEnumerable <TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir); Stopwatch stopwatch = Stopwatch.StartNew(); _frameworkReporter.ReportTestResults(results); stopwatch.Stop(); _testEnvironment.DebugInfo($"Reported {results.Count()} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}"); serializer.UpdateTestDurations(results); } }
public void GetCommandLines_HugeNumberOfSuitesToBeExecutedFully_WontBreakCommandLineCreation() { // we create 400 testsuites, with 380 of them holding 2 test-cases, 20 of them ("MyTestSuite..."[0-19]) holding three test-cases (named Test,Test2,Test3) var range = Enumerable.Range(0, 400); const string baseSuiteName = "MyTestSuiteWithSomeWhatLongNameIsStillHandledCorrectly"; var allTests = range.Select(idx => baseSuiteName + idx + ".Test") .Concat(range.Select(idx => baseSuiteName + idx + ".Test2")) .Concat(range.Take(20).Select(idx => baseSuiteName + idx + ".Test3")) .ToList(); // 305 of the 380 2-test-suites will be executed fully (i.e. all test-cases are executed), only one 3-test-suite is executed fully (which holds also the only Test3-case to be execute) var testsToExecute = allTests.Where((testcase, idx) => idx < 700 || (idx + 5) % 20 == 0).ToList(); IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCasesFull(testsToExecute.ToArray(), allTests.ToArray()); List <CommandLineGenerator.Args> commands = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().ToList(); int longestSuiteName = baseSuiteName.Length + 6; // 6 = 3 for suiteIndex + 3 for suite-filter-delimiter int minimumLineLength = CommandLineGenerator.MaxCommandLength - longestSuiteName; CommandLineGenerator.MaxCommandLength.Should().Be(8191, "if the maximum command line length changes, we may need to adapt the following expectations"); commands.Should().HaveCount(4, "based on given test-cases and maximum command line length we expect 4 command-lines to be created"); commands.First().CommandLine.Should().NotMatchRegex(baseSuiteName + @"(\d|1[^5])\.", "none fo the 3-test-suites (0-19) expect suite 15 must be part of a command line"); commands.First().TestCases.Count(testCase => testCase.DisplayName.EndsWith("Test3")).Should().Be(1, "the single fully executed 3-test-suite has to be part of the first command line"); commands.Skip(1).Select(cmd => cmd.TestCases.Count(testCase => testCase.DisplayName.EndsWith("Test3"))).Sum().Should().Be(0, "only a single Test3 must be executed on the first command line"); commands.Take(3).Select(cmd => cmd.CommandLine.Length).Should().OnlyContain(length => length >= minimumLineLength, "all command lines expect the last need to have a minimum length, so no superfluous command-lines are executed"); commands.Select(cmd => cmd.TestCases.Count).Sum().Should().Be(testsToExecute.Count(), "the number of test to be executed must be identical to those on the command lines"); }
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly() { string userParameters = "-testdirectory=\"MyTestDirectory\""; string commandLine = new CommandLineGenerator(new List<Model.TestCase>(), new List<Model.TestCase>(), TestDataCreator.DummyExecutable.Length, userParameters, "", TestEnvironment).GetCommandLines().First().CommandLine; commandLine.Should().EndWith(" -testdirectory=\"MyTestDirectory\""); }
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly() { string userParameters = "-testdirectory=\"MyTestDirectory\""; string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), new List <Model.TestCase>(), DummyExecutable.Length, userParameters, "", TestEnvironment).GetCommandLines().First().CommandLine; Assert.IsTrue(commandLine.EndsWith(" -testdirectory=\"MyTestDirectory\"")); }
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly() { string userParameters = "-testdirectory=\"MyTestDirectory\""; string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), TestDataCreator.DummyExecutable.Length, userParameters, "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().EndWith(" -testdirectory=\"MyTestDirectory\""); }
public void GetCommandLines_ShuffleTestsWithDefaultSeed_IsAppendedCorrectly() { MockOptions.Setup(o => o.ShuffleTests).Returns(true); IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{GoogleTestConstants.ShuffleTestsOption}"); }
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined() { IEnumerable <Model.TestCase> testCasesWithDifferentSuite = CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest"); IEnumerable <Model.TestCase> allTestCases = testCasesWithDifferentSuite.Union(CreateDummyTestCases("FooSuite.BazTest", "BarSuite.BazTest2")); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithDifferentSuite, DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest", commandLine); }
public void GetCommandLines_TestsWithCommonSuite_AreCombinedViaSuite() { IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest"); IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.FooTest")); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:"); }
public void GetCommandLines_TestsWithCommonSuite_AreCombinedViaSuite() { string[] testCaseNamesWithCommonSuite = { "FooSuite.BarTest", "FooSuite.BazTest" }; string[] allTestCaseNames = testCaseNamesWithCommonSuite.Union("BarSuite.FooTest".Yield()).ToArray(); IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames); string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:"); }
public void GetCommandLines_RepetitionsOption_IsAppendedCorrectly() { MockOptions.Setup(o => o.NrOfTestRepetitions).Returns(4711); IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; string repetitionsOption = GoogleTestConstants.NrOfRepetitionsOption + "=4711"; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{repetitionsOption}"); }
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined() { string[] testCaseNamesWithDifferentSuite = { "BarSuite.BazTest1", "FooSuite.BarTest" }; string[] allTestCaseNames = testCaseNamesWithDifferentSuite.Union(new[] { "FooSuite.BazTest", "BarSuite.BazTest2" }).ToArray(); IEnumerable <Model.TestCase> testCasesWithDifferentSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithDifferentSuite, allTestCaseNames); string commandLine = new CommandLineGenerator(testCasesWithDifferentSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().First().CommandLine; commandLine.Should() .Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest"); }
public void GetCommandLines_BreakOnFailureOption_IsAppendedCorrectly() { IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; string breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(false); commandLine.Should().Contain(breakOnFailureOption); MockOptions.Setup(o => o.BreakOnFailure).Returns(true); commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(true); commandLine.Should().Contain(breakOnFailureOption); }
public void GetCommandLines_ShuffleTestsWithCustomSeed_IsAppendedCorrectly() { MockOptions.Setup(o => o.ShuffleTests).Returns(true); MockOptions.Setup(o => o.ShuffleTestsSeed).Returns(4711); IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; string shuffleTestsOption = GoogleTestConstants.ShuffleTestsOption + GoogleTestConstants.ShuffleTestsSeedOption + "=4711"; Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs}{shuffleTestsOption}", commandLine); }
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite() { IEnumerable <Model.TestCase> testCasesWithCommonSuite = CreateDummyTestCases( "InstantiationName2/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/1"); IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(CreateDummyTestCases("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)")); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0", commandLine); }
public void GetCommandLines_CatchExceptionsOption_IsAppendedCorrectly() { IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; string catchExceptionsOption = GoogleTestConstants.GetCatchExceptionsOption(true); Assert.IsTrue(commandLine.Contains(catchExceptionsOption), commandLine); MockOptions.Setup(o => o.CatchExceptions).Returns(false); commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; catchExceptionsOption = GoogleTestConstants.GetCatchExceptionsOption(false); Assert.IsTrue(commandLine.Contains(catchExceptionsOption), commandLine); }
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly() { List <string> allTests = new List <string>(); List <string> testsToExecute = new List <string>(); for (int i = 0; i < 1000; i++) { allTests.Add("MyTestSuite" + i + ".MyTest"); testsToExecute.Add("MyTestSuite" + i + ".MyTest"); allTests.Add("MyTestSuite" + i + ".MyTest2"); } testsToExecute.Add("MyTestSuite1.MyTest2"); testsToExecute.Add("MyTestSuite5.MyTest2"); IEnumerable <Model.TestCase> allTestCases = allTests.Select(ToTestCase).ToList(); IEnumerable <Model.TestCase> testCases = testsToExecute.Select(ToTestCase).ToList(); List <CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().ToList(); Assert.AreEqual(3, commands.Count); int lengthOfLongestTestname = allTests.Max(s => s.Length); string command = commands[0].CommandLine; Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length); Assert.IsTrue(command.Length >= CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - DummyExecutable.Length - 1); Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:")); command = commands[1].CommandLine; Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length); Assert.IsTrue(command.Length >= CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - DummyExecutable.Length - 1); Assert.IsFalse(command.StartsWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:")); Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=")); command = commands[2].CommandLine; Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length); Assert.IsFalse(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:")); Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=")); HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases); HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases)); Assert.AreEqual(testsAsSet.Count, splittedTestsAsSet.Count); foreach (Model.TestCase testCase in testsAsSet) { Assert.IsTrue(splittedTestsAsSet.Contains(testCase)); } }
public void GetCommandLines_OnlyExitCodeTestCase_DummyFilter() { var exitCodeTestName = "ExitCodeTest"; MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName); var exitCodeTestCase = ExitCodeTestsReporter.CreateExitCodeTestCase(MockOptions.Object, TestDataCreator.DummyExecutable); string commandLine = new CommandLineGenerator(new List <Model.TestCase> { exitCodeTestCase }, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().Contain($"{GoogleTestConstants.FilterOption}GTA_NOT_EXISTING_DUMMY_TEST_CASE"); }
public void GetCommandLines_BreakOnFailureOption_IsAppendedCorrectly() { IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; string breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(false); commandLine.Should().Contain(breakOnFailureOption); MockOptions.Setup(o => o.BreakOnFailure).Returns(true); commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(true); commandLine.Should().Contain(breakOnFailureOption); }
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly() { List <string> allTests = new List <string>(); List <string> testsToExecute = new List <string>(); for (int i = 0; i < 1000; i++) { allTests.Add("MyTestSuite" + i + ".MyTest"); testsToExecute.Add("MyTestSuite" + i + ".MyTest"); allTests.Add("MyTestSuite" + i + ".MyTest2"); } testsToExecute.Add("MyTestSuite1.MyTest2"); testsToExecute.Add("MyTestSuite5.MyTest2"); IEnumerable <Model.TestCase> allTestCases = allTests.Select(TestDataCreator.ToTestCase).ToList(); IEnumerable <Model.TestCase> testCases = testsToExecute.Select(TestDataCreator.ToTestCase).ToList(); List <CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().ToList(); commands.Count.Should().Be(3); int lengthOfLongestTestname = allTests.Max(s => s.Length); int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length; int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1; string commandLine = commands[0].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:"); commandLine = commands[1].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().NotStartWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); commandLine = commands[2].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Should() .NotStartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases); HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases)); splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet); }
public void GetCommandLines_ManyTests_BreaksUpLongCommandLinesCorrectly() { List <string> allTests = new List <string>(); List <string> testsToExecute = new List <string>(); for (int i = 0; i < 1000; i++) { string test1 = $"MyTestSuite{i}.MyTest"; string test2 = $"MyTestSuite{i}.MyTest2"; allTests.Add(test1); testsToExecute.Add(test1); allTests.Add(test2); } IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCasesFull(testsToExecute.ToArray(), allTests.ToArray()); List <CommandLineGenerator.Args> commands = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().ToList(); commands.Count.Should().Be(3); int lengthOfLongestTestname = allTests.Max(s => s.Length); int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length; int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1; string commonStart = $@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite0.MyTest:"; string commandLine = commands[0].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().StartWith(commonStart); commandLine = commands[1].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); commandLine = commands[2].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases); HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases)); splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet); }
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite() { IEnumerable <Model.TestCase> testCasesWithCommonSuite = CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest", "FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb"); IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(CreateDummyTestCases("BarSuite.BarTest")); IEnumerable <Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse(); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; string commandLineFromBackwards = new CommandLineGenerator(allTestCases, testCasesReversed, DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; string ExpectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:"; Assert.AreEqual(ExpectedCommandLine, commandLine); Assert.AreEqual(ExpectedCommandLine, commandLineFromBackwards); }
// ReSharper disable once UnusedParameter.Local private void RunTestsFromExecutable(string executable, string workingDir, IEnumerable <TestCase> testCasesToRun, string userParameters, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher, IProcessExecutor executor) { string resultXmlFile = Path.GetTempFileName(); var serializer = new TestDurationSerializer(); var generator = new CommandLineGenerator(testCasesToRun, executable.Length, userParameters, resultXmlFile, _settings); foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines()) { if (_canceled) { break; } var streamingParser = new StreamingStandardOutputTestResultParser(arguments.TestCases, _logger, _frameworkReporter); var results = RunTests(executable, workingDir, isBeingDebugged, debuggedLauncher, arguments, resultXmlFile, executor, streamingParser).ToArray(); try { Stopwatch stopwatch = Stopwatch.StartNew(); _frameworkReporter.ReportTestsStarted(results.Select(tr => tr.TestCase)); _frameworkReporter.ReportTestResults(results); stopwatch.Stop(); if (results.Length > 0) { _logger.DebugInfo($"{_threadName}Reported {results.Length} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}"); } } catch (TestRunCanceledException e) { _logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}"); Cancel(); } serializer.UpdateTestDurations(results); foreach (TestResult result in results) { if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds)) { _logger.DebugWarning("TestCase already in analyzer: " + result.TestCase.FullyQualifiedName); } } } }
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite() { string[] testCaseNamesWithCommonSuite = { "FooSuite.BarTest", "FooSuite.BazTest", "FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb" }; string[] allTestCaseNames = testCaseNamesWithCommonSuite.Union("BarSuite.BarTest".Yield()).ToArray(); IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames); IEnumerable <Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse(); string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().First().CommandLine; string commandLineFromBackwards = new CommandLineGenerator(testCasesReversed, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().First().CommandLine; string expectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:"; commandLine.Should().Be(expectedCommandLine); commandLineFromBackwards.Should().Be(expectedCommandLine); }
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite() { string[] testCaseNamesWithCommonSuite = { "InstantiationName2/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/1" }; string[] allTestCaseNames = testCaseNamesWithCommonSuite .Union("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)".Yield()) .ToArray(); IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames); string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options) .GetCommandLines().First().CommandLine; commandLine.Should() .Be( $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0"); }
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite() { IEnumerable<Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest", "FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb"); IEnumerable<Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.BarTest")); IEnumerable<Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse(); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; string commandLineFromBackwards = new CommandLineGenerator(allTestCases, testCasesReversed, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; string expectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:"; commandLine.Should().Be(expectedCommandLine); commandLineFromBackwards.Should().Be(expectedCommandLine); }
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined() { IEnumerable<Model.TestCase> testCasesWithDifferentSuite = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest"); IEnumerable<Model.TestCase> allTestCases = testCasesWithDifferentSuite.Union(TestDataCreator.CreateDummyTestCases("FooSuite.BazTest", "BarSuite.BazTest2")); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithDifferentSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; commandLine.Should() .Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest"); }
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly() { List<string> allTests = new List<string>(); List<string> testsToExecute = new List<string>(); for (int i = 0; i < 1000; i++) { allTests.Add("MyTestSuite" + i + ".MyTest"); testsToExecute.Add("MyTestSuite" + i + ".MyTest"); allTests.Add("MyTestSuite" + i + ".MyTest2"); } testsToExecute.Add("MyTestSuite1.MyTest2"); testsToExecute.Add("MyTestSuite5.MyTest2"); IEnumerable<Model.TestCase> allTestCases = allTests.Select(TestDataCreator.ToTestCase).ToList(); IEnumerable<Model.TestCase> testCases = testsToExecute.Select(TestDataCreator.ToTestCase).ToList(); List<CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().ToList(); commands.Count.Should().Be(3); int lengthOfLongestTestname = allTests.Max(s => s.Length); int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length; int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1; string commandLine = commands[0].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:"); commandLine = commands[1].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Length.Should().BeGreaterOrEqualTo(minLength); commandLine.Should().NotStartWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); commandLine = commands[2].CommandLine; commandLine.Length.Should().BeLessThan(maxLength); commandLine.Should() .NotStartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"); commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="); HashSet<Model.TestCase> testsAsSet = new HashSet<Model.TestCase>(testCases); HashSet<Model.TestCase> splittedTestsAsSet = new HashSet<Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases)); splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet); }
public void GetCommandLines_NoTestCases_NoFilter() { string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().NotContain(GoogleTestConstants.FilterOption); }
public void GetCommandLines_RepetitionsOption_IsAppendedCorrectly() { MockOptions.Setup(o => o.NrOfTestRepetitions).Returns(4711); IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; string repetitionsOption = GoogleTestConstants.NrOfRepetitionsOption + "=4711"; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{repetitionsOption}"); }
public void GetCommandLines_ShuffleTestsWithDefaultSeed_IsAppendedCorrectly() { MockOptions.Setup(o => o.ShuffleTests).Returns(true); IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2"); string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine; commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{GoogleTestConstants.ShuffleTestsOption}"); }
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite() { IEnumerable<Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases( "InstantiationName2/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/0", "InstantiationName/ParameterizedTests.SimpleTraits/1"); IEnumerable<Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)")); string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment) .GetCommandLines().First().CommandLine; commandLine.Should() .Be( $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0"); }