private void RunTests(IEnumerable <string> filePaths, bool openInBrowser = false, bool withCodeCoverage = false, bool withDebugger = false) { if (!testingInProgress) { lock (syncLock) { if (!testingInProgress) { dte.Documents.SaveAll(); var solutionDir = Path.GetDirectoryName(dte.Solution.FullName); testingInProgress = true; Task.Factory.StartNew( () => { try { // If setttings file environments have not yet been initialized, do so here. if (settingsEnvironments == null || settingsEnvironments.Count == 0) { InitializeSettingsFileEnvironments(); } var options = new TestOptions { MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism, CoverageOptions = new CoverageOptions { Enabled = withCodeCoverage }, CustomTestLauncher = withDebugger ? new VsDebuggerTestLauncher() : null, TestLaunchMode = GetTestLaunchMode(openInBrowser, withDebugger), ChutzpahSettingsFileEnvironments = settingsEnvironments }; var result = testRunner.RunTests(filePaths, options, runnerCallback); if (result.CoverageObject != null) { var path = CoverageOutputGenerator.WriteHtmlFile(Path.Combine(solutionDir, Constants.CoverageHtmlFileName), result.CoverageObject); processHelper.LaunchFileInBrowser(path); } } catch (Exception e) { Logger.Log("Error while running tests", "ChutzpahPackage", e); } finally { testingInProgress = false; } }); } } } }
private void RunTests(IEnumerable <string> filePaths, bool withCodeCoverage, bool openInBrowser) { if (!testingInProgress) { lock (syncLock) { if (!testingInProgress) { dte.Documents.SaveAll(); var solutionDir = Path.GetDirectoryName(dte.Solution.FullName); testingInProgress = true; Task.Factory.StartNew( () => { try { var options = new TestOptions { TestFileTimeoutMilliseconds = Settings.TimeoutMilliseconds, MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism, CoverageOptions = new CoverageOptions { Enabled = withCodeCoverage }, OpenInBrowser = openInBrowser }; var result = testRunner.RunTests(filePaths, options, runnerCallback); if (result.CoverageObject != null) { var path = CoverageOutputGenerator.WriteHtmlFile(solutionDir, result.CoverageObject); processHelper.LaunchFileInBrowser(path); } } catch (Exception e) { Logger.Log("Error while running tests", "ChutzpahPackage", e); } finally { testingInProgress = false; } }); } } } }
private void ExecuteTestContexts( TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback, ConcurrentBag <TestContext> testContexts, ParallelOptions parallelOptions, string headlessBrowserPath, ConcurrentQueue <TestFileSummary> testFileSummaries, TestCaseSummary overallSummary) { Parallel.ForEach( testContexts, parallelOptions, testContext => { ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode); try { testHarnessBuilder.CreateTestHarness(testContext, options); if (options.OpenInBrowser) { ChutzpahTracer.TraceInformation( "Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.InputTestFile); process.LaunchFileInBrowser(testContext.TestHarnessPath); } else { ChutzpahTracer.TraceInformation( "Invoking headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.InputTestFile); var testSummary = InvokeTestRunner( headlessBrowserPath, options, testContext, testExecutionMode, callback); ChutzpahTracer.TraceInformation( "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors", testContext.TestHarnessPath, testContext.InputTestFile, testSummary.PassedCount, testSummary.FailedCount, testSummary.Errors.Count); ChutzpahTracer.TraceInformation( "Finished running headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.InputTestFile); testFileSummaries.Enqueue(testSummary); } } catch (Exception e) { var error = new TestError { InputTestFile = testContext.InputTestFile, Message = e.ToString() }; overallSummary.Errors.Add(error); callback.FileError(error); ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile); } finally { ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode); } }); // Clean up test context foreach (var testContext in testContexts) { // Don't clean up context if in debug mode if (!m_debugEnabled && !options.OpenInBrowser) { try { ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile); testContextBuilder.CleanupContext(testContext); } catch (Exception e) { ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile); } } } }
private void ExecuteTestContexts( TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback, ConcurrentBag <TestContext> testContexts, ParallelOptions parallelOptions, string headlessBrowserPath, ConcurrentQueue <TestFileSummary> testFileSummaries, TestCaseSummary overallSummary) { Parallel.ForEach( testContexts, parallelOptions, testContext => { ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode); try { try { testHarnessBuilder.CreateTestHarness(testContext, options); } catch (IOException) { // Mark this creation failed so we do not try to clean it up later // This is to work around a bug in TestExplorer that runs chutzpah in parallel on // the same files // TODO(mmanela): Re-evalute if this is needed once they fix that bug testContext.TestHarnessCreationFailed = true; ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile); throw; } if (options.TestLaunchMode == TestLaunchMode.FullBrowser) { ChutzpahTracer.TraceInformation( "Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.FirstInputTestFile); // Allow override from command line. var browserArgs = testContext.TestFileSettings.BrowserArguments; if (!string.IsNullOrWhiteSpace(options.BrowserArgs)) { var path = BrowserPathHelper.GetBrowserPath(options.BrowserName); browserArgs = new Dictionary <string, string> { { Path.GetFileNameWithoutExtension(path), options.BrowserArgs } }; } process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs); } else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser) { ChutzpahTracer.TraceInformation( "Invoking headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.FirstInputTestFile); var testSummaries = InvokeTestRunner( headlessBrowserPath, options, testContext, testExecutionMode, callback); foreach (var testSummary in testSummaries) { ChutzpahTracer.TraceInformation( "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors", testContext.TestHarnessPath, testSummary.Path, testSummary.PassedCount, testSummary.FailedCount, testSummary.Errors.Count); ChutzpahTracer.TraceInformation( "Finished running headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testSummary.Path); testFileSummaries.Enqueue(testSummary); } } else if (options.TestLaunchMode == TestLaunchMode.Custom) { if (options.CustomTestLauncher == null) { throw new ArgumentNullException("TestOptions.CustomTestLauncher"); } ChutzpahTracer.TraceInformation( "Launching custom test on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.FirstInputTestFile); options.CustomTestLauncher.LaunchTest(testContext); } else { Debug.Assert(false); } } catch (Exception e) { var error = new TestError { InputTestFile = testContext.InputTestFiles.FirstOrDefault(), Message = e.ToString() }; overallSummary.Errors.Add(error); callback.FileError(error); ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile); } finally { ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode); } }); // Clean up test context foreach (var testContext in testContexts) { // Don't clean up context if in debug mode if (!m_debugEnabled && !testContext.TestHarnessCreationFailed && options.TestLaunchMode != TestLaunchMode.FullBrowser && options.TestLaunchMode != TestLaunchMode.Custom) { try { ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile); testContextBuilder.CleanupContext(testContext); } catch (Exception e) { ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile); } } } }