private void RunTests_Combine(IEnumerable <TestCase> tests) { if (!tests.Any()) { return; // Sanity check } List <TestCase> groupedtests = new List <TestCase>(); List <TestCase> singledtests = new List <TestCase>(); List <TestCase> remainingtests = new List <TestCase>(); List <TestCase> retrytests = new List <TestCase>(); Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup(); testcasegroup.Source = tests.First().Source; LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}"); // Select tests with the same source foreach (var test in tests) { if (testcasegroup.Source != test.Source) { remainingtests.Add(test); continue; } if (_executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test))) { LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}"); testcasegroup.Names.Add(test.DisplayName); _frameworkHandle.RecordStart(test); // Indicate in the GUI test is running groupedtests.Add(test); } else { singledtests.Add(test); } } // Log sort result LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}"); // Check if source actually exists if (!File.Exists(testcasegroup.Source)) { LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}"); SkipTests(groupedtests); } // Run tests if (_runContext.IsBeingDebugged) { string caselistfilename = _executor.MakeCaselistFilename(testcasegroup.Source); // Prepare testcase list file _executor.CreateTestcaseListFile(testcasegroup, caselistfilename); LogVerbose(TestMessageLevel.Informational, "Start debug run."); _frameworkHandle .LaunchProcessWithDebuggerAttached(testcasegroup.Source , _executor.WorkingDirectory(testcasegroup.Source) , _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename) , _settings.GetEnviromentVariablesForDebug()); // Do not process output in Debug mode foreach (var test in groupedtests) { TestResult result = new TestResult(test); result.Outcome = TestOutcome.None; _frameworkHandle.RecordResult(result); } return; } LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases."); var testresults = _executor.Run(testcasegroup); if (!string.IsNullOrEmpty(_executor.Log)) { LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}"); } // Process results LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}"); foreach (var test in groupedtests) { var testresult = testresults.FindTestResult(test.DisplayName); LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}"); TestResult result = new TestResult(test); if (testresult == null) { if (testresults.TimedOut) { LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}"); result.Outcome = TestOutcome.Skipped; // When test result not found, probably a timeout occured and the test was skipped as a result. result.ErrorMessage = "Timeout of combined testcase execution."; _frameworkHandle.RecordResult(result); } else if (testresults.IsPartialOutput) { LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Looks like it was caused by a previous test crashing the test executable. Adding it to the retry list for another combined test execution run."); retrytests.Add(test); } else { LogNormal(TestMessageLevel.Warning, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Trying again by running it in isolation, i.e., not combined with other test cases. To prevent this try updating to a later version of Catch2 or changing the test case name."); singledtests.Add(test); } } else { RecordTestResult(result, testresult); } } if (retrytests.Count > 0) { LogDebug(TestMessageLevel.Informational, $"Process retry tests (count: {retrytests.Count})"); RunTests_Combine(retrytests); } if (singledtests.Count > 0) { LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})"); RunTests_Single(singledtests); } if (remainingtests.Count > 0) { LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})"); RunTests_Combine(remainingtests); } }
private void RunTests_Combine(IEnumerable <TestCase> tests) { if (!tests.Any()) { return; // Sanity check } List <TestCase> groupedtests = new List <TestCase>(); List <TestCase> singledtests = new List <TestCase>(); List <TestCase> remainingtests = new List <TestCase>(); Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup(); testcasegroup.Source = tests.First().Source; LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}"); // Select tests with the same source foreach (var test in tests) { if (testcasegroup.Source != test.Source) { remainingtests.Add(test); continue; } if (Catch2Interface.Executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test))) { LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}"); testcasegroup.Names.Add(test.DisplayName); _frameworkHandle.RecordStart(test); // Indicate in the GUI test is running groupedtests.Add(test); } else { singledtests.Add(test); } } // Log sort result LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}"); // Check if source actually exists if (!File.Exists(testcasegroup.Source)) { LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}"); SkipTests(groupedtests); } // Run tests LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases."); var testresults = _executor.Run(testcasegroup); if (!string.IsNullOrEmpty(_executor.Log)) { LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}"); } // Process results LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}"); foreach (var test in groupedtests) { var testresult = testresults.FindTestResult(test.DisplayName); LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}"); TestResult result = new TestResult(test); if (testresult == null) { LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}"); result.Outcome = TestOutcome.None; _frameworkHandle.RecordResult(result); singledtests.Add(test); } else { RecordTestResult(result, testresult); } } if (singledtests.Count > 0) { LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})"); RunTests_Single(singledtests); } if (remainingtests.Count > 0) { LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})"); RunTests_Combine(remainingtests); } }