public void RunTests(IEnumerable <TestCase> testCasesToRun, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory)
{
try
{
Stopwatch stopwatch = Stopwatch.StartNew();
string batch = _settings.GetBatchForTestSetup(_testDirectory, _threadId);
SafeRunBatch(TestSetup, _settings.SolutionDir, batch, processExecutorFactory);
_innerTestRunner.RunTests(testCasesToRun, isBeingDebugged, processExecutorFactory);
batch = _settings.GetBatchForTestTeardown(_testDirectory, _threadId);
SafeRunBatch(TestTeardown, _settings.SolutionDir, batch, processExecutorFactory);
stopwatch.Stop();
_logger.DebugInfo($"{_threadName}Execution took {stopwatch.Elapsed}");
if (!Utils.DeleteDirectory(_testDirectory, out var errorMessage))
{
_logger.DebugWarning(
$"{_threadName}Could not delete test directory '" + _testDirectory + "': " + errorMessage);
}
}
catch (Exception e)
{
_logger.LogError($"{_threadName}Exception while running tests: " + e);
}
}
public void RunTests(IEnumerable <TestCase> testCasesToRun, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory)
{
try
{
Stopwatch stopwatch = Stopwatch.StartNew();
string batch = _settings.GetBatchForTestSetup(_testDirectory, _threadId);
SafeRunBatch(BatchType.TestSetup, _settings.SolutionDir, batch, processExecutorFactory);
_innerTestRunner.RunTests(testCasesToRun, isBeingDebugged, processExecutorFactory);
batch = _settings.GetBatchForTestTeardown(_testDirectory, _threadId);
SafeRunBatch(BatchType.TestTeardown, _settings.SolutionDir, batch, processExecutorFactory);
stopwatch.Stop();
_logger.DebugInfo(String.Format(Resources.ExecutionTime, _threadName, stopwatch.Elapsed));
if (!Utils.DeleteDirectory(_testDirectory, out var errorMessage))
{
_logger.DebugWarning(String.Format(Resources.DeleteTestDir, _threadName, _testDirectory, errorMessage));
}
}
catch (Exception e)
{
_logger.LogError(String.Format(Resources.ExceptionMessage, _threadName, e));
}
}
public GoogleTestExecutor(ILogger logger, SettingsWrapper settings, IDebuggedProcessExecutorFactory processExecutorFactory)
{
_logger = logger;
_settings = settings;
_processExecutorFactory = processExecutorFactory;
_schedulingAnalyzer = new SchedulingAnalyzer(logger);
}
private IEnumerable <TestResult> RunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, string resultXmlFile, StreamingStandardOutputTestResultParser streamingParser)
{
try
{
return(TryRunTests(executable, workingDir, isBeingDebugged, processExecutorFactory, arguments, resultXmlFile, streamingParser));
}
catch (Exception e)
{
LogExecutionError(_logger, executable, workingDir, arguments.CommandLine, e);
return(new TestResult[0]);
}
}
private IEnumerable <TestResult> TryRunTests(string executable, string workingDir, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory, CommandLineGenerator.Args arguments, string resultXmlFile,
StreamingStandardOutputTestResultParser streamingParser)
{
var consoleOutput =
RunTestExecutable(executable, workingDir, arguments, isBeingDebugged, processExecutorFactory, streamingParser);
var remainingTestCases =
arguments.TestCases.Except(streamingParser.TestResults.Select(tr => tr.TestCase));
var testResults = new TestResultCollector(_logger, _threadName)
.CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
return(testResults);
}
public void RunTests(IEnumerable <TestCase> testCasesToRun, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory)
{
List <Thread> threads;
lock (this)
{
threads = new List <Thread>();
RunTests(testCasesToRun, threads, isBeingDebugged, processExecutorFactory);
}
foreach (Thread thread in threads)
{
thread.Join();
}
}
// ReSharper disable once UnusedParameter.Local
private void RunTestsFromExecutable(string executable, string workingDir,
IEnumerable <TestCase> testCasesToRun, string userParameters,
bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory)
{
string resultXmlFile = Path.GetTempFileName();
var serializer = new TestDurationSerializer();
var generator = new CommandLineGenerator(testCasesToRun, executable.Length, userParameters, resultXmlFile, _settings);
foreach (CommandLineGenerator.Args arguments in generator.GetCommandLines())
{
if (_canceled)
{
break;
}
var streamingParser = new StreamingStandardOutputTestResultParser(arguments.TestCases, _logger, _frameworkReporter);
var results = RunTests(executable, workingDir, isBeingDebugged, processExecutorFactory, arguments, resultXmlFile, streamingParser).ToArray();
try
{
Stopwatch stopwatch = Stopwatch.StartNew();
_frameworkReporter.ReportTestsStarted(results.Select(tr => tr.TestCase));
_frameworkReporter.ReportTestResults(results);
stopwatch.Stop();
if (results.Length > 0)
{
_logger.DebugInfo($"{_threadName}Reported {results.Length} test results to VS, executable: '{executable}', duration: {stopwatch.Elapsed}");
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
serializer.UpdateTestDurations(results);
foreach (TestResult result in results)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.DebugWarning("TestCase already in analyzer: " + result.TestCase.FullyQualifiedName);
}
}
}
}
public void RunTests(IEnumerable <TestCase> testCasesToRun, bool isBeingDebugged,
IDebuggedProcessExecutorFactory processExecutorFactory)
{
List <Thread> threads;
lock (this)
{
threads = new List <Thread>();
RunTests(testCasesToRun, threads, isBeingDebugged, processExecutorFactory);
}
foreach (Thread thread in threads)
{
thread.Join();
}
// ReSharper disable once InconsistentlySynchronizedField
foreach (var result in _testRunners.SelectMany(r => r.ExecutableResults))
{
ExecutableResults.Add(result);
}
}
private void RunTests(IEnumerable <TestCase> testCasesToRun, List <Thread> threads, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory)
{
TestCase[] testCasesToRunAsArray = testCasesToRun as TestCase[] ?? testCasesToRun.ToArray();
ITestsSplitter splitter = GetTestsSplitter(testCasesToRunAsArray);
List <List <TestCase> > splittedTestCasesToRun = splitter.SplitTestcases();
_logger.LogInfo(string.Format(Resources.ThreadExecutionMessage, splittedTestCasesToRun.Count));
_logger.DebugInfo(Resources.NoTestOutputShown);
int threadId = 0;
foreach (List <TestCase> testcases in splittedTestCasesToRun)
{
var runner = new PreparingTestRunner(threadId++, _frameworkReporter, _logger, _settings.Clone(), _schedulingAnalyzer);
_testRunners.Add(runner);
var thread = new Thread(
() => runner.RunTests(testcases, isBeingDebugged, processExecutorFactory))
{
Name = $"GTA Testrunner {threadId}"
};
threads.Add(thread);
thread.Start();
}
}
private void RunTests(IEnumerable <TestCase> testCasesToRun, List <Thread> threads, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory)
{
TestCase[] testCasesToRunAsArray = testCasesToRun as TestCase[] ?? testCasesToRun.ToArray();
ITestsSplitter splitter = GetTestsSplitter(testCasesToRunAsArray);
List <List <TestCase> > splittedTestCasesToRun = splitter.SplitTestcases();
_logger.LogInfo("Executing tests on " + splittedTestCasesToRun.Count + " threads");
_logger.DebugInfo("Note that no test output will be shown on the test console when executing tests concurrently!");
int threadId = 0;
foreach (List <TestCase> testcases in splittedTestCasesToRun)
{
var runner = new PreparingTestRunner(threadId++, _frameworkReporter, _logger, _settings.Clone(), _schedulingAnalyzer);
_testRunners.Add(runner);
var thread = new Thread(
() => runner.RunTests(testcases, isBeingDebugged, processExecutorFactory))
{
Name = $"GTA Testrunner {threadId}"
};
threads.Add(thread);
thread.Start();
}
}
public void RunTests(IEnumerable <TestCase> testCasesToRun, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory)
{
IDictionary <string, List <TestCase> > groupedTestCases = testCasesToRun.GroupByExecutable();
foreach (string executable in groupedTestCases.Keys)
{
if (_canceled)
{
break;
}
_settings.ExecuteWithSettingsForExecutable(executable, _logger, () =>
{
string workingDir = _settings.GetWorkingDirForExecution(executable, _testDir, _threadId);
string userParameters = _settings.GetUserParametersForExecution(executable, _testDir, _threadId);
RunTestsFromExecutable(
executable,
workingDir,
groupedTestCases[executable],
userParameters,
isBeingDebugged,
processExecutorFactory);
});
}
}
private List <string> RunTestExecutable(string executable, string workingDir, CommandLineGenerator.Args arguments, bool isBeingDebugged, IDebuggedProcessExecutorFactory processExecutorFactory,
StreamingStandardOutputTestResultParser streamingParser)
{
string pathExtension = _settings.GetPathExtension(executable);
bool isTestOutputAvailable = !isBeingDebugged || _settings.DebuggerKind > DebuggerKind.VsTestFramework;
bool printTestOutput = _settings.PrintTestOutput &&
!_settings.ParallelTestExecution &&
isTestOutputAvailable;
void OnNewOutputLine(string line)
{
try
{
if (!_canceled)
{
streamingParser.ReportLine(line);
}
}
catch (TestRunCanceledException e)
{
_logger.DebugInfo($"{_threadName}Execution has been canceled: {e.InnerException?.Message ?? e.Message}");
Cancel();
}
}
_processExecutor = isBeingDebugged
? _settings.DebuggerKind == DebuggerKind.VsTestFramework
? processExecutorFactory.CreateFrameworkDebuggingExecutor(printTestOutput, _logger)
: processExecutorFactory.CreateNativeDebuggingExecutor(
_settings.DebuggerKind == DebuggerKind.Native ? DebuggerEngine.Native : DebuggerEngine.ManagedAndNative,
printTestOutput, _logger)
: processExecutorFactory.CreateExecutor(printTestOutput, _logger);
int exitCode = _processExecutor.ExecuteCommandBlocking(
executable, arguments.CommandLine, workingDir, pathExtension,
isTestOutputAvailable ? (Action <string>)OnNewOutputLine : null);
streamingParser.Flush();
ExecutableResults.Add(new ExecutableResult(executable, exitCode, streamingParser.ExitCodeOutput,
streamingParser.ExitCodeSkip));
var consoleOutput = new List <string>();
new TestDurationSerializer().UpdateTestDurations(streamingParser.TestResults);
_logger.DebugInfo(
$"{_threadName}Reported {streamingParser.TestResults.Count} test results to VS during test execution, executable: '{executable}'");
foreach (TestResult result in streamingParser.TestResults)
{
if (!_schedulingAnalyzer.AddActualDuration(result.TestCase, (int)result.Duration.TotalMilliseconds))
{
_logger.DebugWarning($"{_threadName}TestCase already in analyzer: {result.TestCase.FullyQualifiedName}");
}
}
return(consoleOutput);
}