static Process Start(IFrameworkHandle frameworkHandle, string workingDirectory, string executable, params string[] arguments)
{
var serializedArguments = CommandLine.Serialize(arguments);
if (Debugger.IsAttached)
{
// LaunchProcessWithDebuggerAttached, unlike Process.Start,
// does not automatically propagate environment variables that
// were created within the currently running process, so they
// must be explicitly included here.
var environmentVariables = new Dictionary <string, string>
{
["FIXIE_NAMED_PIPE"] = Environment.GetEnvironmentVariable("FIXIE_NAMED_PIPE")
};
frameworkHandle?
.LaunchProcessWithDebuggerAttached(
executable,
workingDirectory,
serializedArguments,
environmentVariables);
return(null);
}
return(Start(new ProcessStartInfo
{
WorkingDirectory = workingDirectory,
FileName = executable,
Arguments = serializedArguments,
UseShellExecute = false
}));
}
public int ExecuteCommandBlocking(string command, string parameters, string workingDir, string pathExtension,
Action <string> reportOutputLine)
{
if (reportOutputLine != null)
{
throw new ArgumentException(nameof(reportOutputLine));
}
if (_processId.HasValue)
{
throw new InvalidOperationException();
}
IDictionary <string, string> envVariables = new Dictionary <string, string>();
if (!string.IsNullOrEmpty(pathExtension))
{
envVariables["PATH"] = Utils.GetExtendedPath(pathExtension);
}
_logger.DebugInfo($"Attaching debugger to '{command}' via VsTest framework API");
if (_printTestOutput)
{
_logger.DebugInfo(
$"Note that due to restrictions of the VsTest framework, the test executable's output can not be displayed in the test console when debugging tests. Use '{SettingsWrapper.OptionUseNewTestExecutionFramework}' option to overcome this problem.'");
}
_processId = _frameworkHandle.LaunchProcessWithDebuggerAttached(command, workingDir, parameters, envVariables);
ProcessWaiter waiter;
using (var process = Process.GetProcessById(_processId.Value))
{
waiter = new ProcessWaiter(process);
waiter.WaitForExit();
}
return(waiter.ProcessExitCode);
}
static Process Start(IFrameworkHandle frameworkHandle, string workingDirectory, string executable, params string[] arguments)
{
var serializedArguments = CommandLine.Serialize(arguments);
if (Debugger.IsAttached)
{
// LaunchProcessWithDebuggerAttached, unlike Process.Start,
// does not automatically propagate environment variables that
// were created within the currently running process, so they
// must be explicitly included here. It also does not resolve
// bare commands (`dotnet`) to the full file path of the
// corresponding executable, so we must do so manually.
var environmentVariables = new Dictionary <string, string>
{
["FIXIE_NAMED_PIPE"] = Environment.GetEnvironmentVariable("FIXIE_NAMED_PIPE")
};
var filePath = executable == "dotnet" ? FindDotnet() : executable;
frameworkHandle?
.LaunchProcessWithDebuggerAttached(
filePath,
workingDirectory,
serializedArguments,
environmentVariables);
return(null);
}
return(Start(new ProcessStartInfo
{
WorkingDirectory = workingDirectory,
FileName = executable,
Arguments = serializedArguments,
UseShellExecute = false
}));
}
/// <summary>
/// Starts a Process in a debug session using the provided command line arguments string.
/// </summary>
/// <param name="framework">The IFrameworkHandle which provides debug session handling.</param>
/// <param name="info">The process start info which will be used to launch the external test program.</param>
/// <returns>The newly spawned debug process.</returns>
private static Process Debug(IFrameworkHandle framework, ProcessStartInfo info)
{
if (framework == null)
{
return(Run(info));
}
Dictionary <string, string> environment =
info.EnvironmentVariables.Cast <DictionaryEntry>().ToDictionary(
item => item.Key.ToString(),
item => item.Value.ToString()
);
int pid = framework.LaunchProcessWithDebuggerAttached(
info.FileName,
info.WorkingDirectory,
info.Arguments,
environment
);
// Get a process on the local computer, using the process id.
return(Process.GetProcessById(pid));
}
/// <summary>
/// Runs a test in a source (Catch binary) with a debugger attached and returns the resulting XML string.
/// </summary>
internal static string DebugCatchTest(string testBinary, string testSpec, IFrameworkHandle framework)
{
string output;
var cwd = Directory.GetCurrentDirectory();
var exePath = Path.Combine(cwd, testBinary);
var outputPath = Path.GetTempFileName();
try
{
var arguments = Utility.escapeArguments(testSpec, "-r", "xml", "-b", "-d", "yes", "-o", outputPath);
var debuggee = Process.GetProcessById(framework.LaunchProcessWithDebuggerAttached(exePath, cwd, arguments, null));
debuggee.WaitForExit();
output = File.ReadAllText(outputPath);
debuggee.Close();
}
finally
{
File.Delete(outputPath);
}
return(output);
}
public int LaunchProcessWithDebuggerAttached(string command, string workingDirectory, IDictionary <string, string> additionalEnvVars, string param, string pathExtension)
{
IDictionary <string, string> envVariables = new Dictionary <string, string>();
if (additionalEnvVars != null)
{
foreach (var envVar in additionalEnvVars)
{
envVariables[envVar.Key] = envVar.Value;
}
}
if (!string.IsNullOrEmpty(pathExtension))
{
var path = Utils.GetExtendedPath(pathExtension);
if (envVariables.ContainsKey("PATH"))
{
path += $";{envVariables["PATH"]}";
}
envVariables["PATH"] = path;
}
return(_frameworkHandle.LaunchProcessWithDebuggerAttached(command, workingDirectory, param, envVariables));
}
private TestResult RunTest(TestCase test)
{
LogVerbose(TestMessageLevel.Informational, $"Run test: {test.FullyQualifiedName}");
LogDebug(TestMessageLevel.Informational, $"Source: {test.Source}");
LogDebug(TestMessageLevel.Informational, $"SolutionDirectory: {_runContext.SolutionDirectory}");
LogDebug(TestMessageLevel.Informational, $"TestRunDirectory: {_runContext.TestRunDirectory}");
TestResult result = new TestResult(test);
// Check if file exists
if (!File.Exists(test.Source))
{
result.Outcome = TestOutcome.NotFound;
}
// Run test
if (_runContext.IsBeingDebugged)
{
LogVerbose(TestMessageLevel.Informational, "Start debug run.");
_frameworkHandle
.LaunchProcessWithDebuggerAttached(test.Source
, null
, _executor.GenerateCommandlineArguments(test.DisplayName, true)
, null);
// Do not process output in Debug mode
result.Outcome = TestOutcome.None;
}
else
{
var testresult = _executor.Run(test.DisplayName, test.Source);
if (!string.IsNullOrEmpty(_executor.Log))
{
LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
}
// Process test results
if (testresult.TimedOut)
{
LogVerbose(TestMessageLevel.Warning, "Time out");
result.Outcome = TestOutcome.Skipped;
result.ErrorMessage = testresult.ErrorMessage;
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut));
result.Duration = testresult.Duration;
}
else if (testresult.Cancelled)
{
result.Outcome = TestOutcome.None;
}
else
{
result.Outcome = testresult.Success ? TestOutcome.Passed : TestOutcome.Failed;
result.Duration = testresult.Duration;
result.ErrorMessage = testresult.ErrorMessage;
result.ErrorStackTrace = testresult.ErrorStackTrace;
if (testresult.StandardOut != string.Empty)
{
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut));
}
if (testresult.StandardError != string.Empty)
{
result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, testresult.StandardError));
}
}
}
LogVerbose(TestMessageLevel.Informational, $"Finished test: {test.FullyQualifiedName}");
return(result);
}
private void RunTests_Combine(IEnumerable <TestCase> tests)
{
if (!tests.Any())
{
return; // Sanity check
}
List <TestCase> groupedtests = new List <TestCase>();
List <TestCase> singledtests = new List <TestCase>();
List <TestCase> remainingtests = new List <TestCase>();
List <TestCase> retrytests = new List <TestCase>();
Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup();
testcasegroup.Source = tests.First().Source;
LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}");
// Select tests with the same source
foreach (var test in tests)
{
if (testcasegroup.Source != test.Source)
{
remainingtests.Add(test);
continue;
}
if (_executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test)))
{
LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}");
testcasegroup.Names.Add(test.DisplayName);
_frameworkHandle.RecordStart(test); // Indicate in the GUI test is running
groupedtests.Add(test);
}
else
{
singledtests.Add(test);
}
}
// Log sort result
LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}");
// Check if source actually exists
if (!File.Exists(testcasegroup.Source))
{
LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}");
SkipTests(groupedtests);
}
// Run tests
if (_runContext.IsBeingDebugged)
{
string caselistfilename = _executor.MakeCaselistFilename(testcasegroup.Source);
// Prepare testcase list file
_executor.CreateTestcaseListFile(testcasegroup, caselistfilename);
LogVerbose(TestMessageLevel.Informational, "Start debug run.");
_frameworkHandle
.LaunchProcessWithDebuggerAttached(testcasegroup.Source
, _executor.WorkingDirectory(testcasegroup.Source)
, _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename)
, _settings.GetEnviromentVariablesForDebug());
// Do not process output in Debug mode
foreach (var test in groupedtests)
{
TestResult result = new TestResult(test);
result.Outcome = TestOutcome.None;
_frameworkHandle.RecordResult(result);
}
return;
}
LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases.");
var testresults = _executor.Run(testcasegroup);
if (!string.IsNullOrEmpty(_executor.Log))
{
LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
}
// Process results
LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}");
foreach (var test in groupedtests)
{
var testresult = testresults.FindTestResult(test.DisplayName);
LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}");
TestResult result = new TestResult(test);
if (testresult == null)
{
if (testresults.TimedOut)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}");
result.Outcome = TestOutcome.Skipped; // When test result not found, probably a timeout occured and the test was skipped as a result.
result.ErrorMessage = "Timeout of combined testcase execution.";
_frameworkHandle.RecordResult(result);
}
else if (testresults.IsPartialOutput)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Looks like it was caused by a previous test crashing the test executable. Adding it to the retry list for another combined test execution run.");
retrytests.Add(test);
}
else
{
LogNormal(TestMessageLevel.Warning, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Trying again by running it in isolation, i.e., not combined with other test cases. To prevent this try updating to a later version of Catch2 or changing the test case name.");
singledtests.Add(test);
}
}
else
{
RecordTestResult(result, testresult);
}
}
if (retrytests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process retry tests (count: {retrytests.Count})");
RunTests_Combine(retrytests);
}
if (singledtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})");
RunTests_Single(singledtests);
}
if (remainingtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})");
RunTests_Combine(remainingtests);
}
}
/// <summary>
/// Starts a Process in a debug session using the provided command line arguments string.
/// </summary>
/// <param name="framework">The IFrameworkHandle which provides debug session handling.</param>
/// <param name="info">The process start info which will be used to launch the external test program.</param>
/// <returns>The newly spawned debug process.</returns>
private static Process Debug(IFrameworkHandle framework, ProcessStartInfo info)
{
if (framework == null)
{
return Run(info);
}
Dictionary<string, string> environment =
info.EnvironmentVariables.Cast<DictionaryEntry>().ToDictionary(
item => item.Key.ToString(),
item => item.Value.ToString()
);
int pid = framework.LaunchProcessWithDebuggerAttached(
info.FileName,
info.WorkingDirectory,
info.Arguments,
environment
);
// Get a process on the local computer, using the process id.
return Process.GetProcessById(pid);
}
/// <summary>
/// Runs a test in a source (Catch binary) with a debugger attached and returns the resulting XML string.
/// </summary>
internal static string DebugCatchTest(string testBinary, string testSpec, IFrameworkHandle framework)
{
string output;
var cwd = Directory.GetCurrentDirectory();
var exePath = Path.Combine(cwd, testBinary);
var outputPath = Path.GetTempFileName();
try
{
var arguments = Utility.escapeArguments(testSpec, "-r", "xml", "-b", "-d", "yes", "-o", outputPath);
var debuggee = Process.GetProcessById(framework.LaunchProcessWithDebuggerAttached(exePath, cwd, arguments, null));
debuggee.WaitForExit();
output = File.ReadAllText(outputPath);
debuggee.Close();
}
finally
{
File.Delete(outputPath);
}
return output;
}
