private void RunTests_Combine(IEnumerable <TestCase> tests)
{
if (!tests.Any())
{
return; // Sanity check
}
List <TestCase> groupedtests = new List <TestCase>();
List <TestCase> singledtests = new List <TestCase>();
List <TestCase> remainingtests = new List <TestCase>();
List <TestCase> retrytests = new List <TestCase>();
Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup();
testcasegroup.Source = tests.First().Source;
LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}");
// Select tests with the same source
foreach (var test in tests)
{
if (testcasegroup.Source != test.Source)
{
remainingtests.Add(test);
continue;
}
if (_executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test)))
{
LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}");
testcasegroup.Names.Add(test.DisplayName);
_frameworkHandle.RecordStart(test); // Indicate in the GUI test is running
groupedtests.Add(test);
}
else
{
singledtests.Add(test);
}
}
// Log sort result
LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}");
// Check if source actually exists
if (!File.Exists(testcasegroup.Source))
{
LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}");
SkipTests(groupedtests);
}
// Run tests
if (_runContext.IsBeingDebugged)
{
string caselistfilename = _executor.MakeCaselistFilename(testcasegroup.Source);
// Prepare testcase list file
_executor.CreateTestcaseListFile(testcasegroup, caselistfilename);
LogVerbose(TestMessageLevel.Informational, "Start debug run.");
_frameworkHandle
.LaunchProcessWithDebuggerAttached(testcasegroup.Source
, _executor.WorkingDirectory(testcasegroup.Source)
, _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename)
, _settings.GetEnviromentVariablesForDebug());
// Do not process output in Debug mode
foreach (var test in groupedtests)
{
TestResult result = new TestResult(test);
result.Outcome = TestOutcome.None;
_frameworkHandle.RecordResult(result);
}
return;
}
LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases.");
var testresults = _executor.Run(testcasegroup);
if (!string.IsNullOrEmpty(_executor.Log))
{
LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
}
// Process results
LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}");
foreach (var test in groupedtests)
{
var testresult = testresults.FindTestResult(test.DisplayName);
LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}");
TestResult result = new TestResult(test);
if (testresult == null)
{
if (testresults.TimedOut)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}");
result.Outcome = TestOutcome.Skipped; // When test result not found, probably a timeout occured and the test was skipped as a result.
result.ErrorMessage = "Timeout of combined testcase execution.";
_frameworkHandle.RecordResult(result);
}
else if (testresults.IsPartialOutput)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Looks like it was caused by a previous test crashing the test executable. Adding it to the retry list for another combined test execution run.");
retrytests.Add(test);
}
else
{
LogNormal(TestMessageLevel.Warning, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Trying again by running it in isolation, i.e., not combined with other test cases. To prevent this try updating to a later version of Catch2 or changing the test case name.");
singledtests.Add(test);
}
}
else
{
RecordTestResult(result, testresult);
}
}
if (retrytests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process retry tests (count: {retrytests.Count})");
RunTests_Combine(retrytests);
}
if (singledtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})");
RunTests_Single(singledtests);
}
if (remainingtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})");
RunTests_Combine(remainingtests);
}
}
private void RunTests_Combine(IEnumerable <TestCase> tests)
{
if (!tests.Any())
{
return; // Sanity check
}
List <TestCase> groupedtests = new List <TestCase>();
List <TestCase> singledtests = new List <TestCase>();
List <TestCase> remainingtests = new List <TestCase>();
Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup();
testcasegroup.Source = tests.First().Source;
LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}");
// Select tests with the same source
foreach (var test in tests)
{
if (testcasegroup.Source != test.Source)
{
remainingtests.Add(test);
continue;
}
if (Catch2Interface.Executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test)))
{
LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}");
testcasegroup.Names.Add(test.DisplayName);
_frameworkHandle.RecordStart(test); // Indicate in the GUI test is running
groupedtests.Add(test);
}
else
{
singledtests.Add(test);
}
}
// Log sort result
LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}");
// Check if source actually exists
if (!File.Exists(testcasegroup.Source))
{
LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}");
SkipTests(groupedtests);
}
// Run tests
LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases.");
var testresults = _executor.Run(testcasegroup);
if (!string.IsNullOrEmpty(_executor.Log))
{
LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
}
// Process results
LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}");
foreach (var test in groupedtests)
{
var testresult = testresults.FindTestResult(test.DisplayName);
LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}");
TestResult result = new TestResult(test);
if (testresult == null)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}");
result.Outcome = TestOutcome.None;
_frameworkHandle.RecordResult(result);
singledtests.Add(test);
}
else
{
RecordTestResult(result, testresult);
}
}
if (singledtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})");
RunTests_Single(singledtests);
}
if (remainingtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})");
RunTests_Combine(remainingtests);
}
}
