private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, IDictionary <string, string> envVars, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher, CommandLineGenerator.Args arguments, string resultXmlFile, IProcessExecutor executor, StreamingStandardOutputTestResultParser streamingParser) { List <string> consoleOutput; if (_settings.UseNewTestExecutionFramework) { DebugUtils.AssertIsNotNull(executor, nameof(executor)); consoleOutput = RunTestExecutableWithNewFramework(executable, workingDir, envVars, arguments, executor, streamingParser); } else { _processLauncher = new TestProcessLauncher(_logger, _settings, isBeingDebugged); consoleOutput = _processLauncher.GetOutputOfCommand(workingDir, envVars, executable, arguments.CommandLine, _settings.PrintTestOutput && !_settings.ParallelTestExecution, false, debuggedLauncher); } var remainingTestCases = arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase)); var testResults = new TestResultCollector(_logger, _threadName) .CollectTestResults(remainingTestCases, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase); testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList(); return(testResults); }
private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, CommandLineGenerator.Args arguments, IProcessExecutor executor, StreamingStandardOutputTestResultParser streamingParser) { string pathExtension = _settings.GetPathExtension(executable); bool printTestOutput = _settings.PrintTestOutput && !_settings.ParallelTestExecution; if (printTestOutput) { _logger.LogInfo( $"{_threadName}>>>>>>>>>>>>>>> Output of command '" + executable + " " + arguments.CommandLine + "'"); } Action <string> reportOutputAction = line => { try { if (!_canceled) { streamingParser.ReportLine(line); } if (printTestOutput) { _logger.LogInfo(line); } } catch (TestRunCanceledException e) { _logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}"); Cancel(); } }; _processExecutor = executor; _processExecutor.ExecuteCommandBlocking( executable, arguments.CommandLine, workingDir, pathExtension, reportOutputAction); streamingParser.Flush(); if (printTestOutput) { _logger.LogInfo($"{_threadName}<<<<<<<<<<<<<<< End of Output"); } var consoleOutput = new List <string>(); new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults); _logger.DebugInfo( $"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'"); foreach (TestResult result in streamingParser.TestResults) { if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds)) { _logger.LogWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}"); } } return(consoleOutput); }
private List <string> RunTestExecutableWithNewFramework(string executable, string workingDir, IDictionary <string, string> envVars, CommandLineGenerator.Args arguments, IProcessExecutor executor, StreamingStandardOutputTestResultParser streamingParser) { string pathExtension = _settings.GetPathExtension(executable); bool printTestOutput = _settings.PrintTestOutput && !_settings.ParallelTestExecution; if (printTestOutput) { _logger.LogInfo(String.Format(Resources.OutputOfCommandMessage, _threadName, executable, arguments.CommandLine)); } Action <string> reportOutputAction = line => { try { if (!_canceled) { streamingParser.ReportLine(line); } if (printTestOutput) { _logger.LogInfo(line); } } catch (TestRunCanceledException e) { _logger.DebugInfo(String.Format(Resources.ExecutionCancelled, _threadName, e.InnerException?.Message ?? e.Message)); Cancel(); } }; _processExecutor = executor; _processExecutor.ExecuteCommandBlocking( executable, arguments.CommandLine, workingDir, envVars, pathExtension, reportOutputAction); streamingParser.Flush(); if (printTestOutput) { _logger.LogInfo(String.Format(Resources.EndOfOutputMessage, _threadName)); } var consoleOutput = new List <string>(); new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults); _logger.DebugInfo(String.Format(Resources.ReportedResultsToVS, _threadName, streamingParser.TestResults.Count, executable)); foreach (TestResult result in streamingParser.TestResults) { if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds)) { _logger.LogWarning(String.Format(Resources.AlreadyInAnalyzer, _threadName, result.TestCase.FullyQualifiedName)); } } return(consoleOutput); }
private List <string> RunTestExecutable(string executable, string workingDir, CommandLineGenerator.Args arguments, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory, StreamingStandardOutputTestResultParser streamingParser) { string pathExtension = _settings.GetPathExtension(executable); bool isTestOutputAvailable = !isBeingDebugged || _settings.DebuggerKind > DebuggerKind.VsTestFramework; bool printTestOutput = _settings.PrintTestOutput && !_settings.ParallelTestExecution && isTestOutputAvailable; void OnNewOutputLine(string line) { try { if (!_canceled) { streamingParser.ReportLine(line); } } catch (TestRunCanceledException e) { _logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}"); Cancel(); } } _processExecutor = isBeingDebugged ? _settings.DebuggerKind == DebuggerKind.VsTestFramework ? processExecutorFactory.CreateFrameworkDebuggingExecutor(printTestOutput, _logger) : processExecutorFactory.CreateNativeDebuggingExecutor( _settings.DebuggerKind == DebuggerKind.Native ? DebuggerEngine.Native : DebuggerEngine.ManagedAndNative, printTestOutput, _logger) : processExecutorFactory.CreateExecutor(printTestOutput, _logger); int exitCode = _processExecutor.ExecuteCommandBlocking( executable, arguments.CommandLine, workingDir, pathExtension, isTestOutputAvailable ? (Action <string>)OnNewOutputLine : null); streamingParser.Flush(); ExecutableResults.Add(new ExecutableResult(executable, exitCode, streamingParser.ExitCodeOutput, streamingParser.ExitCodeSkip)); var consoleOutput = new List <string>(); new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults); _logger.DebugInfo( $"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'"); foreach (TestResult result in streamingParser.TestResults) { if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds)) { _logger.DebugWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}"); } } return(consoleOutput); }
private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, string resultXmlFile, StreamingStandardOutputTestResultParser streamingParser) { var consoleOutput = RunTestExecutable(executable, workingDir, arguments, isBeingDebugged, processExecutorFactory, streamingParser); var remainingTestCases = arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase)); var testResults = new TestResultCollector(_logger, _threadName) .CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase); testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList(); return(testResults); }
// ReSharper disable once UnusedParameter.Local private void RunTestsFromExecutable(string executable, string workingDir, IEnumerable <TestCase> testCasesToRun, string userParameters, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher, IProcessExecutor executor) { string resultXmlFile = Path.GetTempFileName(); var serializer = new TestDurationSerializer(); var generator = new CommandLineGenerator(testCasesToRun, executable.Length, userParameters, resultXmlFile, _settings); foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines()) { if (_canceled) { break; } var streamingParser = new StreamingStandardOutputTestResultParser(arguments.TestCases, _logger, _frameworkReporter); var results = RunTests(executable, workingDir, isBeingDebugged, debuggedLauncher, arguments, resultXmlFile, executor, streamingParser).ToArray(); try { Stopwatch stopwatch = Stopwatch.StartNew(); _frameworkReporter.ReportTestsStarted(results.Select(tr => tr.TestCase)); _frameworkReporter.ReportTestResults(results); stopwatch.Stop(); if (results.Length > 0) { _logger.DebugInfo($"{_threadName}Reported {results.Length} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}"); } } catch (TestRunCanceledException e) { _logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}"); Cancel(); } serializer.UpdateTestDurations(results); foreach (TestResult result in results) { if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds)) { _logger.DebugWarning("TestCase already in analyzer: " + result.TestCase.FullyQualifiedName); } } } }
private IEnumerable <TestResult> RunTests(string executable, string workingDir, bool isBeingDebugged, IDebuggedProcessLauncher debuggedLauncher, CommandLineGenerator.Args arguments, string resultXmlFile, IProcessExecutor executor, StreamingStandardOutputTestResultParser streamingParser) { try { return(TryRunTests(executable, workingDir, isBeingDebugged, debuggedLauncher, arguments, resultXmlFile, executor, streamingParser)); } catch (Exception e) { LogExecutionError(_logger, executable, workingDir, arguments.CommandLine, e); return(new TestResult[0]); } }
private IEnumerable <TestResult> RunTests(string executable, string workingDir, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, IDictionary <string, string> environmentVariables, string resultXmlFile, StreamingStandardOutputTestResultParser streamingParser) { try { return(TryRunTests(executable, workingDir, isBeingDebugged, processExecutorFactory, arguments, environmentVariables, resultXmlFile, streamingParser)); } catch (Exception e) { LogExecutionError(_logger, executable, workingDir, arguments.CommandLine, e); return(new TestResult[0]); } }