private void DoRunTests(ICollection <TestCase> testCasesToRun, IRunContext runContext, IFrameworkHandle frameworkHandle) { if (testCasesToRun.Count == 0) { return; } bool isRunningInsideVisualStudio = !string.IsNullOrEmpty(runContext.SolutionDirectory); var reporter = new VsTestFrameworkReporter(frameworkHandle, isRunningInsideVisualStudio, _logger); var debuggerAttacher = _debuggerAttacher ?? new MessageBasedDebuggerAttacher(_settings.DebuggingNamedPipeId, _logger); var processExecutorFactory = new DebuggedProcessExecutorFactory(frameworkHandle, debuggerAttacher); var exitCodeTestsAggregator = new ExitCodeTestsAggregator(); var exitCodeTestsReporter = new ExitCodeTestsReporter(reporter, exitCodeTestsAggregator, _settings, _logger); lock (_lock) { if (_canceled) { return; } _executor = new GoogleTestExecutor(_logger, _settings, processExecutorFactory, exitCodeTestsReporter); } _executor.RunTests(testCasesToRun, reporter, runContext.IsBeingDebugged); reporter.AllTestsFinished(); }
private void AssertDurationsFileIsCreated(bool parallelExecution) { string sampleTestsDurationsFile = TestResources.Tests_DebugX86 + GoogleTestConstants.DurationsExtension; RemoveFileIfNecessary(sampleTestsDurationsFile); string crashingTestsDurationsFile = TestResources.CrashingTests_DebugX86 + GoogleTestConstants.DurationsExtension; RemoveFileIfNecessary(crashingTestsDurationsFile); MockOptions.Setup(o => o.ParallelTestExecution).Returns(parallelExecution); MockOptions.Setup(o => o.MaxNrOfThreads).Returns(2); MockOptions.Setup(o => o.SolutionDir).Returns(TestResources.SampleTestsSolutionDir); var collectingReporter = new FakeFrameworkReporter(); var exitCodeTestsAggregator = new ExitCodeTestsAggregator(); var exitCodeTestsReporter = new ExitCodeTestsReporter(MockFrameworkReporter.Object, exitCodeTestsAggregator, MockOptions.Object, MockLogger.Object); var testExecutor = new GoogleTestExecutor(TestEnvironment.Logger, TestEnvironment.Options, ProcessExecutorFactory, exitCodeTestsReporter); testExecutor.RunTests(TestDataCreator.AllTestCasesExceptLoadTests, collectingReporter, false); sampleTestsDurationsFile.AsFileInfo() .Should() .Exist("Test execution should result in test durations"); var durations = new TestDurationSerializer().ReadTestDurations(TestDataCreator.AllTestCasesOfSampleTests); durations.Keys.Should().Contain( TestDataCreator.AllTestCasesOfSampleTests.Where(tc => collectingReporter.ReportedTestResults.Any(tr => tc.Equals(tr.TestCase) && (tr.Outcome == TestOutcome.Passed || tr.Outcome == TestOutcome.Failed)))); crashingTestsDurationsFile.AsFileInfo() .Should() .Exist("Test execution should result in test durations file"); durations = new TestDurationSerializer().ReadTestDurations(TestDataCreator.AllTestCasesOfHardCrashingTests); durations.Keys.Should().Contain(TestDataCreator.AllTestCasesOfHardCrashingTests.Where(tc => collectingReporter.ReportedTestResults.Any(tr => tc.Equals(tr.TestCase) && (tr.Outcome == TestOutcome.Passed || tr.Outcome == TestOutcome.Failed)))); }
public void GetCommandLines_OnlyExitCodeTestCase_DummyFilter() { var exitCodeTestName = "ExitCodeTest"; MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName); var exitCodeTestCase = ExitCodeTestsReporter.CreateExitCodeTestCase(MockOptions.Object, TestDataCreator.DummyExecutable); string commandLine = new CommandLineGenerator(new List <Model.TestCase> { exitCodeTestCase }, TestDataCreator.DummyExecutable.Length, "", "", TestEnvironment.Options).GetCommandLines().First().CommandLine; commandLine.Should().Contain($"{GoogleTestConstants.FilterOption}GTA_NOT_EXISTING_DUMMY_TEST_CASE"); }
public IList <TestCase> CreateTestCases(Action <TestCase> reportTestCase = null) { var standardOutput = new List <string>(); var testCases = new List <TestCase>(); var resolver = new TestCaseResolver(_executable, _diaResolverFactory, _settings, _logger); var suite2TestCases = new Dictionary <string, ISet <TestCase> >(); var parser = new StreamingListTestsParser(_settings.TestNameSeparator); parser.TestCaseDescriptorCreated += (sender, args) => { TestCase testCase; if (_settings.ParseSymbolInformation) { TestCaseLocation testCaseLocation = resolver.FindTestCaseLocation( _signatureCreator.GetTestMethodSignatures(args.TestCaseDescriptor).ToList()); testCase = CreateTestCase(args.TestCaseDescriptor, testCaseLocation); } else { testCase = CreateTestCase(args.TestCaseDescriptor); } testCases.Add(testCase); if (!suite2TestCases.TryGetValue(args.TestCaseDescriptor.Suite, out var testCasesOfSuite)) { suite2TestCases.Add(args.TestCaseDescriptor.Suite, testCasesOfSuite = new HashSet <TestCase>()); } testCasesOfSuite.Add(testCase); }; string workingDir = _settings.GetWorkingDirForDiscovery(_executable); var finalParams = GetDiscoveryParams(); try { int processExitCode = ExecutionFailed; IProcessExecutor executor = null; void OnReportOutputLine(string line) { standardOutput.Add(line); parser.ReportLine(line); } var listAndParseTestsTask = Task.Run(() => { _logger.VerboseInfo($"Starting test discovery for {_executable}"); executor = _processExecutorFactory.CreateExecutor(false, _logger); processExitCode = executor.ExecuteCommandBlocking( _executable, finalParams, workingDir, _settings.GetPathExtension(_executable), OnReportOutputLine); _logger.VerboseInfo($"Finished execution of {_executable}"); }); _logger.VerboseInfo($"Scheduled test discovery for {_executable}"); if (!listAndParseTestsTask.Wait(TimeSpan.FromSeconds(_settings.TestDiscoveryTimeoutInSeconds))) { executor?.Cancel(); LogTimeoutError(workingDir, finalParams, standardOutput); return(new List <TestCase>()); } foreach (var suiteTestCasesPair in suite2TestCases) { foreach (var testCase in suiteTestCasesPair.Value) { testCase.Properties.Add(new TestCaseMetaDataProperty(suiteTestCasesPair.Value.Count, testCases.Count)); reportTestCase?.Invoke(testCase); } } if (!string.IsNullOrWhiteSpace(_settings.ExitCodeTestCase)) { var exitCodeTestCase = ExitCodeTestsReporter.CreateExitCodeTestCase(_settings, _executable, resolver.MainMethodLocation); testCases.Add(exitCodeTestCase); reportTestCase?.Invoke(exitCodeTestCase); _logger.DebugInfo($"Exit code of executable '{_executable}' is ignored for test discovery because option '{SettingsWrapper.OptionExitCodeTestCase}' is set"); } else if (!CheckProcessExitCode(processExitCode, standardOutput, workingDir, finalParams)) { return(new List <TestCase>()); } } catch (Exception e) { SequentialTestRunner.LogExecutionError(_logger, _executable, workingDir, finalParams, e); return(new List <TestCase>()); } return(testCases); }