private void DoRunTests(ICollection <TestCase> testCasesToRun, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
if (testCasesToRun.Count == 0)
{
return;
}
bool isRunningInsideVisualStudio = !string.IsNullOrEmpty(runContext.SolutionDirectory);
var reporter = new VsTestFrameworkReporter(frameworkHandle, isRunningInsideVisualStudio, _logger);
var debuggerAttacher = _debuggerAttacher ?? new MessageBasedDebuggerAttacher(_settings.DebuggingNamedPipeId, _logger);
var processExecutorFactory = new DebuggedProcessExecutorFactory(frameworkHandle, debuggerAttacher);
var exitCodeTestsAggregator = new ExitCodeTestsAggregator();
var exitCodeTestsReporter = new ExitCodeTestsReporter(reporter, exitCodeTestsAggregator, _settings, _logger);
lock (_lock)
{
if (_canceled)
{
return;
}
_executor = new GoogleTestExecutor(_logger, _settings, processExecutorFactory, exitCodeTestsReporter);
}
_executor.RunTests(testCasesToRun, reporter, runContext.IsBeingDebugged);
reporter.AllTestsFinished();
}
private void AssertDurationsFileIsCreated(bool parallelExecution)
{
string sampleTestsDurationsFile = TestResources.Tests_DebugX86 + GoogleTestConstants.DurationsExtension;
RemoveFileIfNecessary(sampleTestsDurationsFile);
string crashingTestsDurationsFile = TestResources.CrashingTests_DebugX86 + GoogleTestConstants.DurationsExtension;
RemoveFileIfNecessary(crashingTestsDurationsFile);
MockOptions.Setup(o => o.ParallelTestExecution).Returns(parallelExecution);
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(2);
MockOptions.Setup(o => o.SolutionDir).Returns(TestResources.SampleTestsSolutionDir);
var collectingReporter = new FakeFrameworkReporter();
var exitCodeTestsAggregator = new ExitCodeTestsAggregator();
var exitCodeTestsReporter = new ExitCodeTestsReporter(MockFrameworkReporter.Object, exitCodeTestsAggregator, MockOptions.Object, MockLogger.Object);
var testExecutor = new GoogleTestExecutor(TestEnvironment.Logger, TestEnvironment.Options, ProcessExecutorFactory, exitCodeTestsReporter);
testExecutor.RunTests(TestDataCreator.AllTestCasesExceptLoadTests, collectingReporter, false);
sampleTestsDurationsFile.AsFileInfo()
.Should()
.Exist("Test execution should result in test durations");
var durations = new TestDurationSerializer().ReadTestDurations(TestDataCreator.AllTestCasesOfSampleTests);
durations.Keys.Should().Contain(
TestDataCreator.AllTestCasesOfSampleTests.Where(tc => collectingReporter.ReportedTestResults.Any(tr => tc.Equals(tr.TestCase) && (tr.Outcome == TestOutcome.Passed || tr.Outcome == TestOutcome.Failed))));
crashingTestsDurationsFile.AsFileInfo()
.Should()
.Exist("Test execution should result in test durations file");
durations = new TestDurationSerializer().ReadTestDurations(TestDataCreator.AllTestCasesOfHardCrashingTests);
durations.Keys.Should().Contain(TestDataCreator.AllTestCasesOfHardCrashingTests.Where(tc => collectingReporter.ReportedTestResults.Any(tr => tc.Equals(tr.TestCase) && (tr.Outcome == TestOutcome.Passed || tr.Outcome == TestOutcome.Failed))));
}
public void GetCommandLines_OnlyExitCodeTestCase_DummyFilter()
{
var exitCodeTestName = "ExitCodeTest";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
var exitCodeTestCase =
ExitCodeTestsReporter.CreateExitCodeTestCase(MockOptions.Object, TestDataCreator.DummyExecutable);
string commandLine = new CommandLineGenerator(new List <Model.TestCase> {
exitCodeTestCase
}, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine;
commandLine.Should().Contain($"{GoogleTestConstants.FilterOption}GTA_NOT_EXISTING_DUMMY_TEST_CASE");
}
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null)
{
var standardOutput = new List <string>();
var testCases = new List <TestCase>();
var resolver = new TestCaseResolver(_executable, _diaResolverFactory, _settings, _logger);
var suite2TestCases = new Dictionary <string, ISet <TestCase> >();
var parser = new StreamingListTestsParser(_settings.TestNameSeparator);
parser.TestCaseDescriptorCreated += (sender, args) =>
{
TestCase testCase;
if (_settings.ParseSymbolInformation)
{
TestCaseLocation testCaseLocation =
resolver.FindTestCaseLocation(
_signatureCreator.GetTestMethodSignatures(args.TestCaseDescriptor).ToList());
testCase = CreateTestCase(args.TestCaseDescriptor, testCaseLocation);
}
else
{
testCase = CreateTestCase(args.TestCaseDescriptor);
}
testCases.Add(testCase);
if (!suite2TestCases.TryGetValue(args.TestCaseDescriptor.Suite, out var testCasesOfSuite))
{
suite2TestCases.Add(args.TestCaseDescriptor.Suite, testCasesOfSuite = new HashSet <TestCase>());
}
testCasesOfSuite.Add(testCase);
};
string workingDir = _settings.GetWorkingDirForDiscovery(_executable);
var finalParams = GetDiscoveryParams();
try
{
int processExitCode = ExecutionFailed;
IProcessExecutor executor = null;
void OnReportOutputLine(string line)
{
standardOutput.Add(line);
parser.ReportLine(line);
}
var listAndParseTestsTask = Task.Run(() =>
{
_logger.VerboseInfo($"Starting test discovery for {_executable}");
executor = _processExecutorFactory.CreateExecutor(false, _logger);
processExitCode = executor.ExecuteCommandBlocking(
_executable,
finalParams,
workingDir,
_settings.GetPathExtension(_executable),
OnReportOutputLine);
_logger.VerboseInfo($"Finished execution of {_executable}");
});
_logger.VerboseInfo($"Scheduled test discovery for {_executable}");
if (!listAndParseTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds)))
{
executor?.Cancel();
LogTimeoutError(workingDir, finalParams, standardOutput);
return(new List <TestCase>());
}
foreach (var suiteTestCasesPair in suite2TestCases)
{
foreach (var testCase in suiteTestCasesPair.Value)
{
testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count));
reportTestCase?.Invoke(testCase);
}
}
if (!string.IsNullOrWhiteSpace(_settings.ExitCodeTestCase))
{
var exitCodeTestCase = ExitCodeTestsReporter.CreateExitCodeTestCase(_settings, _executable, resolver.MainMethodLocation);
testCases.Add(exitCodeTestCase);
reportTestCase?.Invoke(exitCodeTestCase);
_logger.DebugInfo($"Exit code of executable '{_executable}' is ignored for test discovery because option '{SettingsWrapper.OptionExitCodeTestCase}' is set");
}
else if (!CheckProcessExitCode(processExitCode, standardOutput, workingDir, finalParams))
{
return(new List <TestCase>());
}
}
catch (Exception e)
{
SequentialTestRunner.LogExecutionError(_logger, _executable, workingDir, finalParams, e);
return(new List <TestCase>());
}
return(testCases);
}
