// ReSharper disable once UnusedParameter.Local
private void RunTestsFromExecutable(string executable,
IEnumerable <TestCase> allTestCases, IEnumerable <TestCase> testCasesToRun, string baseDir, string userParameters,
bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher)
{
string resultXmlFile = Path.GetTempFileName();
string workingDir = Path.GetDirectoryName(executable);
var serializer = new TestDurationSerializer();
var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, TestEnvironment);
foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines())
{
if (Canceled)
{
break;
}
FrameworkReporter.ReportTestsStarted(arguments.TestCases);
TestEnvironment.DebugInfo("Executing command '" + executable + " " + arguments.CommandLine + "'.");
List <string> consoleOutput = new TestProcessLauncher(TestEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, TestEnvironment.Options.PrintTestOutput && !TestEnvironment.Options.ParallelTestExecution, false, debuggedLauncher);
IEnumerable <TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir);
FrameworkReporter.ReportTestResults(results);
serializer.UpdateTestDurations(results);
}
}
// ReSharper disable once UnusedParameter.Local
private void RunTestsFromExecutable(string executable, string workingDir,
IEnumerable<TestCase> allTestCases, IEnumerable<TestCase> testCasesToRun, string baseDir, string userParameters,
bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher)
{
string resultXmlFile = Path.GetTempFileName();
var serializer = new TestDurationSerializer();
var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, _testEnvironment);
foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines())
{
if (_canceled)
{
break;
}
_frameworkReporter.ReportTestsStarted(arguments.TestCases);
List<string> consoleOutput = new TestProcessLauncher(_testEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, _testEnvironment.Options.PrintTestOutput && !_testEnvironment.Options.ParallelTestExecution, false, debuggedLauncher);
IEnumerable<TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir);
Stopwatch stopwatch = Stopwatch.StartNew();
_frameworkReporter.ReportTestResults(results);
stopwatch.Stop();
_testEnvironment.DebugInfo($"Reported {results.Count()} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}");
serializer.UpdateTestDurations(results);
}
}
public void GetCommandLines_AllTests_ProducesCorrectArguments()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}");
}
public void GetCommandLines_AllTests_ProducesCorrectArguments()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
Assert.AreEqual("--gtest_output=\"xml:\"" + DefaultArgs, commandLine);
}
public void GetCommandLines_AllTests_ProducesCorrectArguments()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1 param", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}");
}
// ReSharper disable once UnusedParameter.Local
private void RunTestsFromExecutable(string executable, string workingDir,
IEnumerable <TestCase> allTestCases, IEnumerable <TestCase> testCasesToRun, string baseDir, string userParameters,
bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher)
{
string resultXmlFile = Path.GetTempFileName();
var serializer = new TestDurationSerializer();
var generator = new CommandLineGenerator(allTestCases, testCasesToRun, executable.Length, userParameters, resultXmlFile, _testEnvironment);
foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines())
{
if (_canceled)
{
break;
}
_frameworkReporter.ReportTestsStarted(arguments.TestCases);
List <string> consoleOutput = new TestProcessLauncher(_testEnvironment, isBeingDebugged).GetOutputOfCommand(workingDir, executable, arguments.CommandLine, _testEnvironment.Options.PrintTestOutput && !_testEnvironment.Options.ParallelTestExecution, false, debuggedLauncher);
IEnumerable <TestResult> results = CollectTestResults(arguments.TestCases, resultXmlFile, consoleOutput, baseDir);
Stopwatch stopwatch = Stopwatch.StartNew();
_frameworkReporter.ReportTestResults(results);
stopwatch.Stop();
_testEnvironment.DebugInfo($"Reported {results.Count()} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}");
serializer.UpdateTestDurations(results);
}
}
public void GetCommandLines_HugeNumberOfSuitesToBeExecutedFully_WontBreakCommandLineCreation()
{
// we create 400 testsuites, with 380 of them holding 2 test-cases, 20 of them ("MyTestSuite..."[0-19]) holding three test-cases (named Test,Test2,Test3)
var range = Enumerable.Range(0, 400);
const string baseSuiteName = "MyTestSuiteWithSomeWhatLongNameIsStillHandledCorrectly";
var allTests = range.Select(idx => baseSuiteName + idx + ".Test")
.Concat(range.Select(idx => baseSuiteName + idx + ".Test2"))
.Concat(range.Take(20).Select(idx => baseSuiteName + idx + ".Test3"))
.ToList();
// 305 of the 380 2-test-suites will be executed fully (i.e. all test-cases are executed), only one 3-test-suite is executed fully (which holds also the only Test3-case to be execute)
var testsToExecute = allTests.Where((testcase, idx) => idx < 700 || (idx + 5) % 20 == 0).ToList();
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCasesFull(testsToExecute.ToArray(), allTests.ToArray());
List <CommandLineGenerator.Args> commands =
new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().ToList();
int longestSuiteName = baseSuiteName.Length + 6; // 6 = 3 for suiteIndex + 3 for suite-filter-delimiter
int minimumLineLength = CommandLineGenerator.MaxCommandLength - longestSuiteName;
CommandLineGenerator.MaxCommandLength.Should().Be(8191, "if the maximum command line length changes, we may need to adapt the following expectations");
commands.Should().HaveCount(4, "based on given test-cases and maximum command line length we expect 4 command-lines to be created");
commands.First().CommandLine.Should().NotMatchRegex(baseSuiteName + @"(\d|1[^5])\.", "none fo the 3-test-suites (0-19) expect suite 15 must be part of a command line");
commands.First().TestCases.Count(testCase => testCase.DisplayName.EndsWith("Test3")).Should().Be(1, "the single fully executed 3-test-suite has to be part of the first command line");
commands.Skip(1).Select(cmd => cmd.TestCases.Count(testCase => testCase.DisplayName.EndsWith("Test3"))).Sum().Should().Be(0, "only a single Test3 must be executed on the first command line");
commands.Take(3).Select(cmd => cmd.CommandLine.Length).Should().OnlyContain(length => length >= minimumLineLength, "all command lines expect the last need to have a minimum length, so no superfluous command-lines are executed");
commands.Select(cmd => cmd.TestCases.Count).Sum().Should().Be(testsToExecute.Count(), "the number of test to be executed must be identical to those on the command lines");
}
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly()
{
string userParameters = "-testdirectory=\"MyTestDirectory\"";
string commandLine = new CommandLineGenerator(new List<Model.TestCase>(), new List<Model.TestCase>(), TestDataCreator.DummyExecutable.Length, userParameters, "", TestEnvironment).GetCommandLines().First().CommandLine;
commandLine.Should().EndWith(" -testdirectory=\"MyTestDirectory\"");
}
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly()
{
string userParameters = "-testdirectory=\"MyTestDirectory\"";
string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), new List <Model.TestCase>(), DummyExecutable.Length, userParameters, "", TestEnvironment).GetCommandLines().First().CommandLine;
Assert.IsTrue(commandLine.EndsWith(" -testdirectory=\"MyTestDirectory\""));
}
public void GetCommandLines_AdditionalArguments_AreAppendedCorrectly()
{
string userParameters = "-testdirectory=\"MyTestDirectory\"";
string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), TestDataCreator.DummyExecutable.Length, userParameters, "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().EndWith(" -testdirectory=\"MyTestDirectory\"");
}
public void GetCommandLines_ShuffleTestsWithDefaultSeed_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.ShuffleTests).Returns(true);
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{GoogleTestConstants.ShuffleTestsOption}");
}
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined()
{
IEnumerable <Model.TestCase> testCasesWithDifferentSuite = CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest");
IEnumerable <Model.TestCase> allTestCases = testCasesWithDifferentSuite.Union(CreateDummyTestCases("FooSuite.BazTest", "BarSuite.BazTest2"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithDifferentSuite, DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest", commandLine);
}
public void GetCommandLines_TestsWithCommonSuite_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.FooTest"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:");
}
public void GetCommandLines_TestsWithCommonSuite_AreCombinedViaSuite()
{
string[] testCaseNamesWithCommonSuite = { "FooSuite.BarTest", "FooSuite.BazTest" };
string[] allTestCaseNames = testCaseNamesWithCommonSuite.Union("BarSuite.FooTest".Yield()).ToArray();
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames);
string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:");
}
public void GetCommandLines_RepetitionsOption_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.NrOfTestRepetitions).Returns(4711);
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
string repetitionsOption = GoogleTestConstants.NrOfRepetitionsOption + "=4711";
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{repetitionsOption}");
}
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined()
{
string[] testCaseNamesWithDifferentSuite = { "BarSuite.BazTest1", "FooSuite.BarTest" };
string[] allTestCaseNames = testCaseNamesWithDifferentSuite.Union(new[] { "FooSuite.BazTest", "BarSuite.BazTest2" }).ToArray();
IEnumerable <Model.TestCase> testCasesWithDifferentSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithDifferentSuite, allTestCaseNames);
string commandLine = new CommandLineGenerator(testCasesWithDifferentSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest");
}
public void GetCommandLines_BreakOnFailureOption_IsAppendedCorrectly()
{
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
string breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(false);
commandLine.Should().Contain(breakOnFailureOption);
MockOptions.Setup(o => o.BreakOnFailure).Returns(true);
commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(true);
commandLine.Should().Contain(breakOnFailureOption);
}
public void GetCommandLines_ShuffleTestsWithCustomSeed_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.ShuffleTests).Returns(true);
MockOptions.Setup(o => o.ShuffleTestsSeed).Returns(4711);
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
string shuffleTestsOption = GoogleTestConstants.ShuffleTestsOption
+ GoogleTestConstants.ShuffleTestsSeedOption + "=4711";
Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs}{shuffleTestsOption}", commandLine);
}
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = CreateDummyTestCases(
"InstantiationName2/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/1");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(CreateDummyTestCases("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
Assert.AreEqual($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0", commandLine);
}
public void GetCommandLines_CatchExceptionsOption_IsAppendedCorrectly()
{
IEnumerable <Model.TestCase> testCases = CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
string catchExceptionsOption = GoogleTestConstants.GetCatchExceptionsOption(true);
Assert.IsTrue(commandLine.Contains(catchExceptionsOption), commandLine);
MockOptions.Setup(o => o.CatchExceptions).Returns(false);
commandLine = new CommandLineGenerator(testCases, testCases, DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
catchExceptionsOption = GoogleTestConstants.GetCatchExceptionsOption(false);
Assert.IsTrue(commandLine.Contains(catchExceptionsOption), commandLine);
}
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly()
{
List <string> allTests = new List <string>();
List <string> testsToExecute = new List <string>();
for (int i = 0; i < 1000; i++)
{
allTests.Add("MyTestSuite" + i + ".MyTest");
testsToExecute.Add("MyTestSuite" + i + ".MyTest");
allTests.Add("MyTestSuite" + i + ".MyTest2");
}
testsToExecute.Add("MyTestSuite1.MyTest2");
testsToExecute.Add("MyTestSuite5.MyTest2");
IEnumerable <Model.TestCase> allTestCases = allTests.Select(ToTestCase).ToList();
IEnumerable <Model.TestCase> testCases = testsToExecute.Select(ToTestCase).ToList();
List <CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().ToList();
Assert.AreEqual(3, commands.Count);
int lengthOfLongestTestname = allTests.Max(s => s.Length);
string command = commands[0].CommandLine;
Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length);
Assert.IsTrue(command.Length >= CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - DummyExecutable.Length - 1);
Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:"));
command = commands[1].CommandLine;
Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length);
Assert.IsTrue(command.Length >= CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - DummyExecutable.Length - 1);
Assert.IsFalse(command.StartsWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"));
Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="));
command = commands[2].CommandLine;
Assert.IsTrue(command.Length < CommandLineGenerator.MaxCommandLength - DummyExecutable.Length);
Assert.IsFalse(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:"));
Assert.IsTrue(command.StartsWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter="));
HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases);
HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases));
Assert.AreEqual(testsAsSet.Count, splittedTestsAsSet.Count);
foreach (Model.TestCase testCase in testsAsSet)
{
Assert.IsTrue(splittedTestsAsSet.Contains(testCase));
}
}
public void GetCommandLines_OnlyExitCodeTestCase_DummyFilter()
{
var exitCodeTestName = "ExitCodeTest";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
var exitCodeTestCase =
ExitCodeTestsReporter.CreateExitCodeTestCase(MockOptions.Object, TestDataCreator.DummyExecutable);
string commandLine = new CommandLineGenerator(new List <Model.TestCase> {
exitCodeTestCase
}, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Contain($"{GoogleTestConstants.FilterOption}GTA_NOT_EXISTING_DUMMY_TEST_CASE");
}
public void GetCommandLines_BreakOnFailureOption_IsAppendedCorrectly()
{
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
string breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(false);
commandLine.Should().Contain(breakOnFailureOption);
MockOptions.Setup(o => o.BreakOnFailure).Returns(true);
commandLine = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
breakOnFailureOption = GoogleTestConstants.GetBreakOnFailureOption(true);
commandLine.Should().Contain(breakOnFailureOption);
}
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly()
{
List <string> allTests = new List <string>();
List <string> testsToExecute = new List <string>();
for (int i = 0; i < 1000; i++)
{
allTests.Add("MyTestSuite" + i + ".MyTest");
testsToExecute.Add("MyTestSuite" + i + ".MyTest");
allTests.Add("MyTestSuite" + i + ".MyTest2");
}
testsToExecute.Add("MyTestSuite1.MyTest2");
testsToExecute.Add("MyTestSuite5.MyTest2");
IEnumerable <Model.TestCase> allTestCases = allTests.Select(TestDataCreator.ToTestCase).ToList();
IEnumerable <Model.TestCase> testCases = testsToExecute.Select(TestDataCreator.ToTestCase).ToList();
List <CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().ToList();
commands.Count.Should().Be(3);
int lengthOfLongestTestname = allTests.Max(s => s.Length);
int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length;
int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1;
string commandLine = commands[0].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:");
commandLine = commands[1].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().NotStartWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:");
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
commandLine = commands[2].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Should()
.NotStartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:");
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases);
HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases));
splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet);
}
public void GetCommandLines_ManyTests_BreaksUpLongCommandLinesCorrectly()
{
List <string> allTests = new List <string>();
List <string> testsToExecute = new List <string>();
for (int i = 0; i < 1000; i++)
{
string test1 = $"MyTestSuite{i}.MyTest";
string test2 = $"MyTestSuite{i}.MyTest2";
allTests.Add(test1);
testsToExecute.Add(test1);
allTests.Add(test2);
}
IEnumerable <Model.TestCase> testCases = TestDataCreator.CreateDummyTestCasesFull(testsToExecute.ToArray(), allTests.ToArray());
List <CommandLineGenerator.Args> commands = new CommandLineGenerator(testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().ToList();
commands.Count.Should().Be(3);
int lengthOfLongestTestname = allTests.Max(s => s.Length);
int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length;
int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1;
string commonStart = $@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite0.MyTest:";
string commandLine = commands[0].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().StartWith(commonStart);
commandLine = commands[1].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
commandLine = commands[2].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
HashSet <Model.TestCase> testsAsSet = new HashSet <Model.TestCase>(testCases);
HashSet <Model.TestCase> splittedTestsAsSet = new HashSet <Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases));
splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet);
}
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite()
{
IEnumerable <Model.TestCase> testCasesWithCommonSuite = CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest",
"FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb");
IEnumerable <Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(CreateDummyTestCases("BarSuite.BarTest"));
IEnumerable <Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse();
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string commandLineFromBackwards = new CommandLineGenerator(allTestCases, testCasesReversed, DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string ExpectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:";
Assert.AreEqual(ExpectedCommandLine, commandLine);
Assert.AreEqual(ExpectedCommandLine, commandLineFromBackwards);
}
// ReSharper disable once UnusedParameter.Local
private void RunTestsFromExecutable(string executable, string workingDir,
IEnumerable <TestCase> testCasesToRun, string userParameters,
bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher, IProcessExecutor executor)
{
string resultXmlFile = Path.GetTempFileName();
var serializer = new TestDurationSerializer();
var generator = new CommandLineGenerator(testCasesToRun, executable.Length, userParameters, resultXmlFile, _settings);
foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines())
{
if (_canceled)
{
break;
}
var streamingParser = new StreamingStandardOutputTestResultParser(arguments.TestCases, _logger, _frameworkReporter);
var results = RunTests(executable, workingDir, isBeingDebugged, debuggedLauncher, arguments, resultXmlFile, executor, streamingParser).ToArray();
try
{
Stopwatch stopwatch = Stopwatch.StartNew();
_frameworkReporter.ReportTestsStarted(results.Select(tr => tr.TestCase));
_frameworkReporter.ReportTestResults(results);
stopwatch.Stop();
if (results.Length > 0)
{
_logger.DebugInfo($"{_threadName}Reported {results.Length} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}");
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
serializer.UpdateTestDurations(results);
foreach (TestResult result in results)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.DebugWarning("TestCase already in analyzer: " + result.TestCase.FullyQualifiedName);
}
}
}
}
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite()
{
string[] testCaseNamesWithCommonSuite = { "FooSuite.BarTest", "FooSuite.BazTest",
"FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb" };
string[] allTestCaseNames = testCaseNamesWithCommonSuite.Union("BarSuite.BarTest".Yield()).ToArray();
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames);
IEnumerable <Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse();
string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
string commandLineFromBackwards = new CommandLineGenerator(testCasesReversed, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
string expectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:";
commandLine.Should().Be(expectedCommandLine);
commandLineFromBackwards.Should().Be(expectedCommandLine);
}
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite()
{
string[] testCaseNamesWithCommonSuite =
{
"InstantiationName2/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/1"
};
string[] allTestCaseNames = testCaseNamesWithCommonSuite
.Union("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)".Yield())
.ToArray();
IEnumerable <Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCasesFull(testCaseNamesWithCommonSuite, allTestCaseNames);
string commandLine = new CommandLineGenerator(testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be(
$"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0");
}
public void GetCommandLines_TestsWithCommonSuiteInReverseOrder_AreCombinedViaSuite()
{
IEnumerable<Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases("FooSuite.BarTest", "FooSuite.BazTest",
"FooSuite.gsdfgdfgsdfg", "FooSuite.23453452345", "FooSuite.bxcvbxcvbxcvb");
IEnumerable<Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("BarSuite.BarTest"));
IEnumerable<Model.TestCase> testCasesReversed = testCasesWithCommonSuite.Reverse();
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string commandLineFromBackwards = new CommandLineGenerator(allTestCases, testCasesReversed, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
string expectedCommandLine = $"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=FooSuite.*:";
commandLine.Should().Be(expectedCommandLine);
commandLineFromBackwards.Should().Be(expectedCommandLine);
}
public void GetCommandLines_TestsWithoutCommonSuiteInDifferentOrder_AreNotCombined()
{
IEnumerable<Model.TestCase> testCasesWithDifferentSuite = TestDataCreator.CreateDummyTestCases("BarSuite.BazTest1", "FooSuite.BarTest");
IEnumerable<Model.TestCase> allTestCases = testCasesWithDifferentSuite.Union(TestDataCreator.CreateDummyTestCases("FooSuite.BazTest", "BarSuite.BazTest2"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithDifferentSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be($"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=BarSuite.BazTest1:FooSuite.BarTest");
}
public void GetCommandLines_ManyTestsWithSuites_BreaksUpLongCommandLinesCorrectly()
{
List<string> allTests = new List<string>();
List<string> testsToExecute = new List<string>();
for (int i = 0; i < 1000; i++)
{
allTests.Add("MyTestSuite" + i + ".MyTest");
testsToExecute.Add("MyTestSuite" + i + ".MyTest");
allTests.Add("MyTestSuite" + i + ".MyTest2");
}
testsToExecute.Add("MyTestSuite1.MyTest2");
testsToExecute.Add("MyTestSuite5.MyTest2");
IEnumerable<Model.TestCase> allTestCases = allTests.Select(TestDataCreator.ToTestCase).ToList();
IEnumerable<Model.TestCase> testCases = testsToExecute.Select(TestDataCreator.ToTestCase).ToList();
List<CommandLineGenerator.Args> commands = new CommandLineGenerator(allTestCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().ToList();
commands.Count.Should().Be(3);
int lengthOfLongestTestname = allTests.Max(s => s.Length);
int maxLength = CommandLineGenerator.MaxCommandLength - TestDataCreator.DummyExecutable.Length;
int minLength = CommandLineGenerator.MaxCommandLength - lengthOfLongestTestname - TestDataCreator.DummyExecutable.Length - 1;
string commandLine = commands[0].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:MyTestSuite0.MyTest:");
commandLine = commands[1].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Length.Should().BeGreaterOrEqualTo(minLength);
commandLine.Should().NotStartWith(@"--gtest_output=""xml:"" --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:");
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
commandLine = commands[2].CommandLine;
commandLine.Length.Should().BeLessThan(maxLength);
commandLine.Should()
.NotStartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=MyTestSuite1.*:MyTestSuite5.*:");
commandLine.Should().StartWith($@"--gtest_output=""xml:""{DefaultArgs} --gtest_filter=");
HashSet<Model.TestCase> testsAsSet = new HashSet<Model.TestCase>(testCases);
HashSet<Model.TestCase> splittedTestsAsSet = new HashSet<Model.TestCase>(commands[0].TestCases.Union(commands[1].TestCases).Union(commands[2].TestCases));
splittedTestsAsSet.Should().BeEquivalentTo(testsAsSet);
}
public void GetCommandLines_NoTestCases_NoFilter()
{
string commandLine = new CommandLineGenerator(new List <Model.TestCase>(), TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().NotContain(GoogleTestConstants.FilterOption);
}
public void GetCommandLines_RepetitionsOption_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.NrOfTestRepetitions).Returns(4711);
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
string repetitionsOption = GoogleTestConstants.NrOfRepetitionsOption + "=4711";
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{repetitionsOption}");
}
public void GetCommandLines_ShuffleTestsWithDefaultSeed_IsAppendedCorrectly()
{
MockOptions.Setup(o => o.ShuffleTests).Returns(true);
IEnumerable<Model.TestCase> testCases = TestDataCreator.CreateDummyTestCases("Suite1.Test1", "Suite2.Test2");
string commandLine = new CommandLineGenerator(testCases, testCases, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment).GetCommandLines().First().CommandLine;
commandLine.Should().Be($"--gtest_output=\"xml:\"{DefaultArgs}{GoogleTestConstants.ShuffleTestsOption}");
}
public void GetCommandLines_ParameterizedTestsWithCommonSuite_AreCombinedViaSuite()
{
IEnumerable<Model.TestCase> testCasesWithCommonSuite = TestDataCreator.CreateDummyTestCases(
"InstantiationName2/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/0",
"InstantiationName/ParameterizedTests.SimpleTraits/1");
IEnumerable<Model.TestCase> allTestCases = testCasesWithCommonSuite.Union(TestDataCreator.CreateDummyTestCases("InstantiationName2/ParameterizedTests.SimpleTraits/1 # GetParam() = (,2)"));
string commandLine = new CommandLineGenerator(allTestCases, testCasesWithCommonSuite, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment)
.GetCommandLines().First().CommandLine;
commandLine.Should()
.Be(
$"--gtest_output=\"xml:\"{DefaultArgs} --gtest_filter=InstantiationName/ParameterizedTests.*:InstantiationName2/ParameterizedTests.SimpleTraits/0");
}