public void ExceptionThrown(Exception exception, string fileName) { lock (sync) { nestedCallback.ExceptionThrown(exception, fileName); } }
private bool PerformBatchCompile(ITestMethodRunnerCallback callback, IEnumerable <TestContext> testContexts) { try { batchCompilerService.Compile(testContexts); } catch (FileNotFoundException e) { callback.ExceptionThrown(e); return(false); } catch (ChutzpahCompilationFailedException e) { callback.ExceptionThrown(e, e.SettingsFile); ChutzpahTracer.TraceError(e, "Error during batch compile from {0}", e.SettingsFile); return(false); } return(true); }
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths, TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback, IChutzpahWebServerHost activeWebServerHost = null) { var overallSummary = new TestCaseSummary(); options.TestExecutionMode = testExecutionMode; options.DebugEnabled = m_debugEnabled; stopWatch.Start(); if (testPaths == null) { throw new ArgumentNullException("testPaths"); } if (fileProbe.FindFilePath(TestRunnerJsName) == null) { throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName); } // Concurrent list to collect test contexts var testContexts = new ConcurrentBag <TestContext>(); // Concurrent collection used to gather the parallel results from var testFileSummaries = new ConcurrentQueue <TestFileSummary>(); var resultCount = 0; var cancellationSource = new CancellationTokenSource(); try { // Given the input paths discover the potential test files var scriptPaths = FindTestFiles(testPaths, options); // Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled. // If so, we keep those tests in a group together to be used in one context // Otherwise, we put each file in its own test group so each get their own context var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options); ConfigureTracing(testRunConfiguration); var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue ? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value) : options.MaxDegreeOfParallelism; var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token }; ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism); // Build test contexts in parallel given a list of files each BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary); // Compile the test contexts if (!PerformBatchCompile(callback, testContexts)) { return(overallSummary); } // Find the first test context with a web server configuration and use it var webServerHost = SetupWebServerHost(testContexts, options); ActiveWebServerHost = webServerHost; // Build test harness for each context and execute it in parallel ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, testFileSummaries, overallSummary, webServerHost); // Gather TestFileSummaries into TaseCaseSummary foreach (var fileSummary in testFileSummaries) { overallSummary.Append(fileSummary); } stopWatch.Stop(); overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds); overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary); ChutzpahTracer.TraceInformation( "Chutzpah run finished with {0} passed, {1} failed and {2} errors", overallSummary.PassedCount, overallSummary.FailedCount, overallSummary.Errors.Count); return(overallSummary); } catch (Exception e) { callback.ExceptionThrown(e); ChutzpahTracer.TraceError(e, "Unhandled exception during Chutzpah test run"); return(overallSummary); } finally { // Clear the settings file cache since in VS Chutzpah is not unloaded from memory. // If we don't clear then the user can never update the file. testSettingsService.ClearCache(); } }
private bool PerformBatchCompile(ITestMethodRunnerCallback callback, IEnumerable<TestContext> testContexts) { try { batchCompilerService.Compile(testContexts); } catch (ChutzpahCompilationFailedException e) { callback.ExceptionThrown(e, e.SettingsFile); ChutzpahTracer.TraceError(e, "Error during batch compile from {0}", e.SettingsFile); return false; } return true; }
/// <inheritdoc/> public void ExceptionThrown(string assemblyFilename, Exception exception) { callback.ExceptionThrown(testClass.TestAssembly, exception); }
public void Compile(IEnumerable <TestContext> testContexts, ITestMethodRunnerCallback callback = null) { // Group the test contexts by test settings to run batch aware settings like compile // For each test settings file that defines a compile step we will run it and update // testContexts reference files accordingly. var groupedTestContexts = testContexts.GroupBy(x => x.TestFileSettings); foreach (var contextGroup in groupedTestContexts) { var testSettings = contextGroup.Key; // If there is no compile setting then nothing to do here if (testSettings.Compile == null) { continue; } // Build the mapping from source to output files and gather properties about them var filePropeties = ( from file in contextGroup.SelectMany(x => x.ReferencedFiles).Where(x => !x.IsBuiltInDependency).Distinct() where testSettings.Compile.Extensions.Any(x => file.Path.EndsWith(x, StringComparison.OrdinalIgnoreCase)) let sourceProperties = GetFileProperties(file.Path) let sourceHasOutput = !testSettings.Compile.ExtensionsWithNoOutput.Any(x => file.Path.EndsWith(x, StringComparison.OrdinalIgnoreCase)) let outputPath = GetOutputPath(file.Path, testSettings.Compile) let outputProperties = sourceHasOutput ? GetFileProperties(outputPath) : null select new SourceCompileInfo { SourceProperties = sourceProperties, OutputProperties = outputProperties, SourceHasOutput = sourceHasOutput }).ToList(); var outputPathMap = filePropeties .Where(x => x.SourceHasOutput) .ToDictionary(x => x.SourceProperties.Path, x => x.OutputProperties.Path, StringComparer.OrdinalIgnoreCase); // Check if the batch compile is needed var shouldCompile = CheckIfCompileIsNeeded(testSettings, filePropeties); // Run the batch compile if necessary if (shouldCompile) { if (testSettings.Compile.Mode == BatchCompileMode.Executable) { RunBatchCompile(testSettings); } else { ChutzpahTracer.TraceWarning("Chutzpah determined generated .js files are missing but the compile mode is External so Chutzpah can't compile them. Test results may be wrong."); } } else { ChutzpahTracer.TraceInformation("Skipping batch compile since all files are update to date for {0}", testSettings.SettingsFileName); } // Now that compile finished set generated path on all files who match the compiled extensions var filesToUpdate = contextGroup.SelectMany(x => x.ReferencedFiles) .Where(x => outputPathMap.ContainsKey(x.Path)); foreach (var file in filesToUpdate) { var outputPath = outputPathMap[file.Path]; if (outputPath != null && fileSystem.FileExists(outputPath)) { file.GeneratedFilePath = outputPath; ChutzpahTracer.TraceInformation("Found generated path for {0} at {1}", file.Path, outputPath); } else { // If we could not find the file at the configured path attempt to see if it co-located ChutzpahTracer.TraceInformation("Unable to find generated path at configured location so attempting to see if generated file is co-located."); var coLocatedOutputPath = Path.ChangeExtension(file.Path, ".js"); if (fileSystem.FileExists(coLocatedOutputPath)) { file.GeneratedFilePath = coLocatedOutputPath; ChutzpahTracer.TraceInformation("Found generated path for {0} at {1}", file.Path, coLocatedOutputPath); } else { var error = string.Format("Couldn't find generated path for {0} at {1} or at {2}", file.Path, outputPath, coLocatedOutputPath); ChutzpahTracer.TraceError(error); if (!testSettings.Compile.IgnoreMissingFiles.GetValueOrDefault()) { // Throw and fail here since if we cant find the file we cannot be sure anything will run var exception = new FileNotFoundException(error, outputPath); callback.ExceptionThrown(exception, outputPath); } } } if (!string.IsNullOrWhiteSpace(file.GeneratedFilePath)) { file.SourceMapFilePath = testSettings.Compile.UseSourceMaps.GetValueOrDefault() ? sourceMapDiscoverer.FindSourceMap(file.GeneratedFilePath) : null; } } } }
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths, TestOptions options, TestRunnerMode testRunnerMode, ITestMethodRunnerCallback callback) { stopWatch.Start(); string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName); if (testPaths == null) throw new ArgumentNullException("testPaths"); if (headlessBrowserPath == null) throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName); if (fileProbe.FindFilePath(TestRunnerJsName) == null) throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName); var overallSummary = new TestCaseSummary(); // Concurrent collection used to gather the parallel results from var testFileSummaries = new ConcurrentQueue<TestFileSummary>(); var resultCount = 0; var cancellationSource = new CancellationTokenSource(); var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token }; Parallel.ForEach(fileProbe.FindScriptFiles(testPaths, options.TestingMode), parallelOptions, testFile => { try { if (cancellationSource.IsCancellationRequested) return; TestContext testContext; resultCount++; if (testContextBuilder.TryBuildContext(testFile, options, out testContext)) { if (options.OpenInBrowser) { ChutzpahTracer.TraceInformation("Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.InputTestFile); process.LaunchFileInBrowser(testContext.TestHarnessPath); } else { ChutzpahTracer.TraceInformation("Invoking test runner on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.InputTestFile); var testSummary = InvokeTestRunner(headlessBrowserPath, options, testContext, testRunnerMode, callback); testFileSummaries.Enqueue(testSummary); } if (!m_debugEnabled && !options.OpenInBrowser) { ChutzpahTracer.TraceInformation("Cleaning up test context artifacts"); // Don't clean up context if in debug mode testContextBuilder.CleanupContext(testContext); } } // Limit the number of files we can scan to attempt to build a context for // This is important in the case of folder scanning where many JS files may not be // test files. if (resultCount >= options.FileSearchLimit) { cancellationSource.Cancel(); } } catch (Exception e) { callback.ExceptionThrown(e, testFile.FullPath); } }); // Gather TestFileSummaries into TaseCaseSummary foreach (var fileSummary in testFileSummaries) { overallSummary.Append(fileSummary); } stopWatch.Stop(); overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds); compilerCache.Save(); return overallSummary; }
public void ExceptionThrown(TestAssembly testAssembly, Exception exception) { innerCallback.ExceptionThrown(testAssembly, exception); }