public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
List <string> standardOutput = new List <string>();
if (_settings.UseNewTestExecutionFramework)
{
return(NewCreateTestcases(reportTestCase, standardOutput));
}
try
{
var launcher = new ProcessLauncher(_logger, _settings.GetPathExtension(_executable), null);
int processExitCode;
string workingDir = new FileInfo(_executable).DirectoryName;
standardOutput = launcher.GetOutputOfCommand(workingDir, null, _executable, GoogleTestConstants.ListTestsOption,
false, false, out processExitCode);
if (!CheckProcessExitCode(processExitCode, standardOutput))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, Path.GetFullPath(""),
GoogleTestConstants.ListTestsOption, e);
return(new List <TestCase>());
}
IList <TestCaseDescriptor> testCaseDescriptors = new ListTestsParser(_settings.TestNameSeparator).ParseListTestsOutput(standardOutput);
var testCaseLocations = GetTestCaseLocations(testCaseDescriptors, _settings.GetPathExtension(_executable));
IList <TestCase> testCases = new List <TestCase>();
IDictionary <string, ISet <TestCase> > suite2TestCases = new Dictionary <string, ISet <TestCase> >();
foreach (var descriptor in testCaseDescriptors)
{
var testCase = _settings.ParseSymbolInformation
? CreateTestCase(descriptor, testCaseLocations)
: CreateTestCase(descriptor);
ISet <TestCase> testCasesInSuite;
if (!suite2TestCases.TryGetValue(descriptor.Suite, out testCasesInSuite))
{
suite2TestCases.Add(descriptor.Suite, testCasesInSuite = new HashSet <TestCase>());
}
testCasesInSuite.Add(testCase);
testCases.Add(testCase);
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
}
}
return(testCases);
}
private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, CommandLineGenerator.Args arguments, IProcessExecutor executor,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution;
if (printTestOutput)
{
_logger.LogInfo(
$"{_threadName}>>>>>>>>>>>>>>> Output of command '" + executable + " " + arguments.CommandLine + "'");
}
Action <string> reportOutputAction = line =>
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
if (printTestOutput)
{
_logger.LogInfo(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
};
_processExecutor = executor;
_processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, pathExtension,
reportOutputAction);
streamingParser.Flush();
if (printTestOutput)
{
_logger.LogInfo($"{_threadName}<<<<<<<<<<<<<<< End of Output");
}
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(
$"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.LogWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
}
}
return(consoleOutput);
}
private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, IDictionary <string, string> envVars,
CommandLineGenerator.Args arguments, IProcessExecutor executor,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution;
if (printTestOutput)
{
_logger.LogInfo(String.Format(Resources.OutputOfCommandMessage, _threadName, executable, arguments.CommandLine));
}
Action <string> reportOutputAction = line =>
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
if (printTestOutput)
{
_logger.LogInfo(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo(String.Format(Resources.ExecutionCancelled, _threadName, e.InnerException?.Message ?? e.Message));
Cancel();
}
};
_processExecutor = executor;
_processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, envVars, pathExtension,
reportOutputAction);
streamingParser.Flush();
if (printTestOutput)
{
_logger.LogInfo(String.Format(Resources.EndOfOutputMessage, _threadName));
}
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(String.Format(Resources.ReportedResultsToVS, _threadName, streamingParser.TestResults.Count, executable));
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.LogWarning(String.Format(Resources.AlreadyInAnalyzer, _threadName, result.TestCase.FullyQualifiedName));
}
}
return(consoleOutput);
}
private List <string> RunTestExecutable(string executable, string workingDir, CommandLineGenerator.Args arguments, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool isTestOutputAvailable = !isBeingDebugged || _settings.DebuggerKind > DebuggerKind.VsTestFramework;
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution &&
isTestOutputAvailable;
void OnNewOutputLine(string line)
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
}
_processExecutor = isBeingDebugged
? _settings.DebuggerKind == DebuggerKind.VsTestFramework
? processExecutorFactory.CreateFrameworkDebuggingExecutor(printTestOutput, _logger)
: processExecutorFactory.CreateNativeDebuggingExecutor(
_settings.DebuggerKind == DebuggerKind.Native ? DebuggerEngine.Native : DebuggerEngine.ManagedAndNative,
printTestOutput, _logger)
: processExecutorFactory.CreateExecutor(printTestOutput, _logger);
int exitCode = _processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, pathExtension,
isTestOutputAvailable ? (Action <string>)OnNewOutputLine : null);
streamingParser.Flush();
ExecutableResults.Add(new ExecutableResult(executable, exitCode, streamingParser.ExitCodeOutput,
streamingParser.ExitCodeSkip));
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(
$"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.DebugWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
}
}
return(consoleOutput);
}
private void AddSymbolsFromBinary(string binary, bool resolveMainMethod = false)
{
string pdb = PdbLocator.FindPdbFile(binary, _settings.GetPathExtension(_executable), _logger);
if (pdb == null)
{
_logger.DebugWarning($"No .pdb file found for '{binary}'");
return;
}
AddSymbolsFromBinary(binary, pdb, resolveMainMethod);
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
List <string> standardOutput = new List <string>();
if (_settings.UseNewTestExecutionFramework)
{
return(NewCreateTestcases(reportTestCase, standardOutput));
}
try
{
var launcher = new ProcessLauncher(_logger, _settings.GetPathExtension(_executable));
int processExitCode;
standardOutput = launcher.GetOutputOfCommand("", _executable, GoogleTestConstants.ListTestsOption.Trim(),
false, false, out processExitCode);
if (!CheckProcessExitCode(processExitCode, standardOutput))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, Path.GetFullPath(""),
GoogleTestConstants.ListTestsOption.Trim(), e);
return(new List <TestCase>());
}
IList <TestCaseDescriptor> testCaseDescriptors = new ListTestsParser(_settings.TestNameSeparator).ParseListTestsOutput(standardOutput);
if (_settings.ParseSymbolInformation)
{
List <TestCaseLocation> testCaseLocations = GetTestCaseLocations(testCaseDescriptors, _settings.GetPathExtension(_executable));
return(testCaseDescriptors.Select(descriptor => CreateTestCase(descriptor, testCaseLocations)).ToList());
}
return(testCaseDescriptors.Select(CreateTestCase).ToList());
}
public List <string> GetOutputOfCommand(string workingDirectory, IDictionary <string, string> envVars, string command, string param, bool printTestOutput,
bool throwIfError, IDebuggedProcessLauncher debuggedLauncher, out int processExitCode)
{
if (_isBeingDebugged)
{
var output = new List <string>();
processExitCode = LaunchProcessWithDebuggerAttached(workingDirectory, envVars, command, param, printTestOutput, debuggedLauncher);
return(output);
}
var actualLauncher = new ProcessLauncher(_logger, _settings.GetPathExtension(command), processId => _processId = processId);
return(actualLauncher.GetOutputOfCommand(workingDirectory, envVars, command, param, printTestOutput,
throwIfError, out processExitCode));
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
var standardOutput = new List <string>();
var testCases = new List <TestCase>();
var resolver = new TestCaseResolver(_executable, _diaResolverFactory, _settings, _logger);
var suite2TestCases = new Dictionary <string, ISet <TestCase> >();
var parser = new StreamingListTestsParser(_settings.TestNameSeparator);
parser.TestCaseDescriptorCreated += (sender, args) =>
{
TestCase testCase;
if (_settings.ParseSymbolInformation)
{
TestCaseLocation testCaseLocation =
resolver.FindTestCaseLocation(
_signatureCreator.GetTestMethodSignatures(args.TestCaseDescriptor).ToList());
testCase = CreateTestCase(args.TestCaseDescriptor, testCaseLocation);
}
else
{
testCase = CreateTestCase(args.TestCaseDescriptor);
}
testCases.Add(testCase);
if (!suite2TestCases.TryGetValue(args.TestCaseDescriptor.Suite, out var testCasesOfSuite))
{
suite2TestCases.Add(args.TestCaseDescriptor.Suite, testCasesOfSuite = new HashSet <TestCase>());
}
testCasesOfSuite.Add(testCase);
};
string workingDir = _settings.GetWorkingDirForDiscovery(_executable);
var finalParams = GetDiscoveryParams();
try
{
int processExitCode = ExecutionFailed;
IProcessExecutor executor = null;
void OnReportOutputLine(string line)
{
standardOutput.Add(line);
parser.ReportLine(line);
}
var listAndParseTestsTask = Task.Run(() =>
{
_logger.VerboseInfo($"Starting test discovery for {_executable}");
executor = _processExecutorFactory.CreateExecutor(false, _logger);
processExitCode = executor.ExecuteCommandBlocking(
_executable,
finalParams,
workingDir,
_settings.GetPathExtension(_executable),
OnReportOutputLine);
_logger.VerboseInfo($"Finished execution of {_executable}");
});
_logger.VerboseInfo($"Scheduled test discovery for {_executable}");
if (!listAndParseTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
{
executor?.Cancel();
LogTimeoutError(workingDir, finalParams, standardOutput);
return(new List <TestCase>());
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
reportTestCase?.Invoke(testCase);
}
}
if (!string.IsNullOrWhiteSpace(_settings.ExitCodeTestCase))
{
var exitCodeTestCase = ExitCodeTestsReporter.CreateExitCodeTestCase(_settings, _executable, resolver.MainMethodLocation);
testCases.Add(exitCodeTestCase);
reportTestCase?.Invoke(exitCodeTestCase);
_logger.DebugInfo($"Exit code of executable '{_executable}' is ignored for test discovery because option '{SettingsWrapper.OptionExitCodeTestCase}' is set");
}
else if (!CheckProcessExitCode(processExitCode, standardOutput, workingDir, finalParams))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, workingDir, finalParams, e);
return(new List <TestCase>());
}
return(testCases);
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
if (_settings.UseNewTestExecutionFramework)
{
return(NewCreateTestcases(reportTestCase));
}
string workingDir = _settings.GetWorkingDirForDiscovery(_executable);
string finalParams = GetDiscoveryParams();
List <string> standardOutput = new List <string>();
try
{
int processExitCode = 0;
ProcessLauncher launcher = null;
var listTestsTask = new Task(() =>
{
launcher = new ProcessLauncher(_logger, _settings.GetPathExtension(_executable), null);
standardOutput = launcher.GetOutputOfCommand(workingDir, _executable, finalParams,
false, false, out processExitCode);
}, TaskCreationOptions.AttachedToParent);
listTestsTask.Start();
if (!listTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
{
launcher?.Cancel();
LogTimeoutError(workingDir, finalParams);
return(new List <TestCase>());
}
if (!CheckProcessExitCode(processExitCode, standardOutput, workingDir, finalParams))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, workingDir, finalParams, e);
return(new List <TestCase>());
}
IList <TestCaseDescriptor> testCaseDescriptors = new ListTestsParser(_settings.TestNameSeparator).ParseListTestsOutput(standardOutput);
var resolver = new TestCaseResolver(_executable, _settings.GetPathExtension(_executable), _settings.GetAdditionalPdbs(_executable), _diaResolverFactory, _settings.ParseSymbolInformation, _logger);
IList <TestCase> testCases = new List <TestCase>();
IDictionary <string, ISet <TestCase> > suite2TestCases = new Dictionary <string, ISet <TestCase> >();
foreach (var descriptor in testCaseDescriptors)
{
var testCase = _settings.ParseSymbolInformation
? CreateTestCase(descriptor, resolver)
: CreateTestCase(descriptor);
ISet <TestCase> testCasesInSuite;
if (!suite2TestCases.TryGetValue(descriptor.Suite, out testCasesInSuite))
{
suite2TestCases.Add(descriptor.Suite, testCasesInSuite = new HashSet <TestCase>());
}
testCasesInSuite.Add(testCase);
testCases.Add(testCase);
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
}
}
if (reportTestCase != null)
{
foreach (var testCase in testCases)
{
reportTestCase(testCase);
}
}
return(testCases);
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
List <string> standardOutput = new List <string>();
if (_settings.UseNewTestExecutionFramework)
{
return(NewCreateTestcases(reportTestCase, standardOutput));
}
try
{
int processExitCode = 0;
ProcessLauncher launcher = null;
var listTestsTask = new Task(() =>
{
launcher = new ProcessLauncher(_logger, _settings.GetPathExtension(_executable), null);
standardOutput = launcher.GetOutputOfCommand("", _executable, GoogleTestConstants.ListTestsOption,
false, false, out processExitCode);
}, TaskCreationOptions.AttachedToParent);
listTestsTask.Start();
if (!listTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
{
launcher?.Cancel();
string dir = Path.GetDirectoryName(_executable);
string file = Path.GetFileName(_executable);
string cdToWorkingDir = $@"cd ""{dir}""";
string listTestsCommand = $"{file} {GoogleTestConstants.ListTestsOption}";
_logger.LogError($"Test discovery was cancelled after {_settings.TestDiscoveryTimeoutInSeconds}s for executable {_executable}");
_logger.DebugError($"Test whether the following commands can be executed sucessfully on the command line (make sure all required binaries are on the PATH):{Environment.NewLine}{cdToWorkingDir}{Environment.NewLine}{listTestsCommand}");
return(new List <TestCase>());
}
if (!CheckProcessExitCode(processExitCode, standardOutput))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, Path.GetFullPath(""),
GoogleTestConstants.ListTestsOption, e);
return(new List <TestCase>());
}
IList <TestCaseDescriptor> testCaseDescriptors = new ListTestsParser(_settings.TestNameSeparator).ParseListTestsOutput(standardOutput);
var resolver = new TestCaseResolver(_executable, _settings.GetPathExtension(_executable), _diaResolverFactory, _settings.ParseSymbolInformation, _logger);
IList <TestCase> testCases = new List <TestCase>();
IDictionary <string, ISet <TestCase> > suite2TestCases = new Dictionary <string, ISet <TestCase> >();
foreach (var descriptor in testCaseDescriptors)
{
var testCase = _settings.ParseSymbolInformation
? CreateTestCase(descriptor, resolver)
: CreateTestCase(descriptor);
ISet <TestCase> testCasesInSuite;
if (!suite2TestCases.TryGetValue(descriptor.Suite, out testCasesInSuite))
{
suite2TestCases.Add(descriptor.Suite, testCasesInSuite = new HashSet <TestCase>());
}
testCasesInSuite.Add(testCase);
testCases.Add(testCase);
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
}
}
if (reportTestCase != null)
{
foreach (var testCase in testCases)
{
reportTestCase(testCase);
}
}
return(testCases);
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
var standardOutput = new List <string>();
var testCases = new List <TestCase>();
var resolver = new TestCaseResolver(
_executable,
_settings.GetPathExtension(_executable),
_settings.GetAdditionalPdbs(_executable),
_diaResolverFactory,
_settings.ParseSymbolInformation,
_logger);
var suite2TestCases = new Dictionary <string, ISet <TestCase> >();
var parser = new StreamingListTestsParser(_settings.TestNameSeparator);
parser.TestCaseDescriptorCreated += (sender, args) =>
{
TestCase testCase;
if (_settings.ParseSymbolInformation)
{
TestCaseLocation testCaseLocation =
resolver.FindTestCaseLocation(
_signatureCreator.GetTestMethodSignatures(args.TestCaseDescriptor).ToList());
testCase = CreateTestCase(args.TestCaseDescriptor, testCaseLocation);
}
else
{
testCase = CreateTestCase(args.TestCaseDescriptor);
}
testCases.Add(testCase);
ISet <TestCase> testCasesOfSuite;
if (!suite2TestCases.TryGetValue(args.TestCaseDescriptor.Suite, out testCasesOfSuite))
{
suite2TestCases.Add(args.TestCaseDescriptor.Suite, testCasesOfSuite = new HashSet <TestCase>());
}
testCasesOfSuite.Add(testCase);
};
string workingDir = _settings.GetWorkingDirForDiscovery(_executable);
var finalParams = GetDiscoveryParams();
try
{
int processExitCode = ExecutionFailed;
IProcessExecutor executor = null;
void OnReportOutputLine(string line)
{
standardOutput.Add(line);
parser.ReportLine(line);
}
var listAndParseTestsTask = new Task(() =>
{
executor = _processExecutorFactory.CreateExecutor(false, _logger);
processExitCode = executor.ExecuteCommandBlocking(
_executable,
finalParams,
workingDir,
_settings.GetPathExtension(_executable),
OnReportOutputLine);
}, TaskCreationOptions.AttachedToParent);
listAndParseTestsTask.Start();
if (!listAndParseTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
{
executor?.Cancel();
LogTimeoutError(workingDir, finalParams, standardOutput);
return(new List <TestCase>());
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
reportTestCase?.Invoke(testCase);
}
}
if (!CheckProcessExitCode(processExitCode, standardOutput, workingDir, finalParams))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, workingDir, finalParams, e);
return(new List <TestCase>());
}
return(testCases);
}
