private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, IDictionary <string, string> envVars,
CommandLineGenerator.Args arguments, IProcessExecutor executor,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution;
if (printTestOutput)
{
_logger.LogInfo(String.Format(Resources.OutputOfCommandMessage, _threadName, executable, arguments.CommandLine));
}
Action <string> reportOutputAction = line =>
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
if (printTestOutput)
{
_logger.LogInfo(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo(String.Format(Resources.ExecutionCancelled, _threadName, e.InnerException?.Message ?? e.Message));
Cancel();
}
};
_processExecutor = executor;
_processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, envVars, pathExtension,
reportOutputAction);
streamingParser.Flush();
if (printTestOutput)
{
_logger.LogInfo(String.Format(Resources.EndOfOutputMessage, _threadName));
}
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(String.Format(Resources.ReportedResultsToVS, _threadName, streamingParser.TestResults.Count, executable));
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.LogWarning(String.Format(Resources.AlreadyInAnalyzer, _threadName, result.TestCase.FullyQualifiedName));
}
}
return(consoleOutput);
}
private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, CommandLineGenerator.Args arguments, IProcessExecutor executor,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution;
if (printTestOutput)
{
_logger.LogInfo(
$"{_threadName}>>>>>>>>>>>>>>> Output of command '" + executable + " " + arguments.CommandLine + "'");
}
Action <string> reportOutputAction = line =>
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
if (printTestOutput)
{
_logger.LogInfo(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
};
_processExecutor = executor;
_processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, pathExtension,
reportOutputAction);
streamingParser.Flush();
if (printTestOutput)
{
_logger.LogInfo($"{_threadName}<<<<<<<<<<<<<<< End of Output");
}
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(
$"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.LogWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
}
}
return(consoleOutput);
}
private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessLauncher debuggedLauncher, CommandLineGenerator.Args arguments, string resultXmlFile, IProcessExecutor executor,
StreamingStandardOutputTestResultParser streamingParser)
{
List <string> consoleOutput;
if (_settings.UseNewTestExecutionFramework)
{
DebugUtils.AssertIsNotNull(executor, nameof(executor));
consoleOutput = RunTestExecutableWithNewFramework(executable, workingDir, arguments, executor, streamingParser);
}
else
{
_processLauncher = new TestProcessLauncher(_logger, _settings, isBeingDebugged);
consoleOutput =
_processLauncher.GetOutputOfCommand(workingDir, executable, arguments.CommandLine,
_settings.PrintTestOutput && !_settings.ParallelTestExecution, false,
debuggedLauncher);
}
var remainingTestCases =
arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase));
var testResults = new TestResultCollector(_logger, _threadName)
.CollectTestResults(remainingTestCases, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
return(testResults);
}
private IEnumerable <TestResult> RunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessLauncher debuggedLauncher, CommandLineGenerator.Args arguments, string resultXmlFile, IProcessExecutor executor, StreamingStandardOutputTestResultParser streamingParser)
{
try
{
return(TryRunTests(executable, workingDir, isBeingDebugged, debuggedLauncher, arguments, resultXmlFile, executor, streamingParser));
}
catch (Exception e)
{
LogExecutionError(_logger, executable, workingDir, arguments.CommandLine, e);
return(new TestResult[0]);
}
}
private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, string baseDir, bool isBeingDebugged,
IDebuggedProcessLauncher debuggedLauncher, CommandLineGenerator.Args arguments, string resultXmlFile, IProcessExecutor executor,
TestResultSplitter splitter)
{
List <string> consoleOutput;
if (_settings.UseNewTestExecutionFramework)
{
DebugUtils.AssertIsNotNull(executor, nameof(executor));
consoleOutput = RunTestExecutableWithNewFramework(executable, workingDir, arguments, executor, splitter);
}
else
{
consoleOutput =
new TestProcessLauncher(_logger, _settings, isBeingDebugged)
.GetOutputOfCommand(workingDir, executable, arguments.CommandLine,
_settings.PrintTestOutput && !_settings.ParallelTestExecution, false,
debuggedLauncher);
}
var remainingTestCases =
arguments.TestCases.Except(splitter.TestResults.Select(tr => tr.TestCase));
return(CollectTestResults(remainingTestCases, resultXmlFile, consoleOutput, baseDir, splitter.CrashedTestCase));
}
private List <string> RunTestExecutable(string executable, string workingDir, CommandLineGenerator.Args arguments, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool isTestOutputAvailable = !isBeingDebugged || _settings.DebuggerKind > DebuggerKind.VsTestFramework;
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution &&
isTestOutputAvailable;
void OnNewOutputLine(string line)
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
}
_processExecutor = isBeingDebugged
? _settings.DebuggerKind == DebuggerKind.VsTestFramework
? processExecutorFactory.CreateFrameworkDebuggingExecutor(printTestOutput, _logger)
: processExecutorFactory.CreateNativeDebuggingExecutor(
_settings.DebuggerKind == DebuggerKind.Native ? DebuggerEngine.Native : DebuggerEngine.ManagedAndNative,
printTestOutput, _logger)
: processExecutorFactory.CreateExecutor(printTestOutput, _logger);
int exitCode = _processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, pathExtension,
isTestOutputAvailable ? (Action <string>)OnNewOutputLine : null);
streamingParser.Flush();
ExecutableResults.Add(new ExecutableResult(executable, exitCode, streamingParser.ExitCodeOutput,
streamingParser.ExitCodeSkip));
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(
$"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.DebugWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
}
}
return(consoleOutput);
}
private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, string resultXmlFile,
StreamingStandardOutputTestResultParser streamingParser)
{
var consoleOutput =
RunTestExecutable(executable, workingDir, arguments, isBeingDebugged, processExecutorFactory, streamingParser);
var remainingTestCases =
arguments.TestCases
.Except(streamingParser.TestResults.Select(tr => tr.TestCase))
.Where(tc => !tc.IsExitCodeTestCase);
var testResults = new TestResultCollector(_logger, _threadName, _settings)
.CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
return(testResults);
}
private IEnumerable <TestResult> RunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, IDictionary <string, string> environmentVariables, string resultXmlFile, StreamingStandardOutputTestResultParser streamingParser)
{
try
{
return(TryRunTests(executable, workingDir, isBeingDebugged, processExecutorFactory, arguments, environmentVariables, resultXmlFile, streamingParser));
}
catch (Exception e)
{
LogExecutionError(_logger, executable, workingDir, arguments.CommandLine, e);
return(new TestResult[0]);
}
}