public void ExceptionThrown(Exception exception, string fileName)
{
lock (sync)
{
nestedCallback.ExceptionThrown(exception, fileName);
}
}
private bool PerformBatchCompile(ITestMethodRunnerCallback callback, IEnumerable <TestContext> testContexts)
{
try
{
batchCompilerService.Compile(testContexts);
}
catch (FileNotFoundException e)
{
callback.ExceptionThrown(e);
return(false);
}
catch (ChutzpahCompilationFailedException e)
{
callback.ExceptionThrown(e, e.SettingsFile);
ChutzpahTracer.TraceError(e, "Error during batch compile from {0}", e.SettingsFile);
return(false);
}
return(true);
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
IChutzpahWebServerHost activeWebServerHost = null)
{
var overallSummary = new TestCaseSummary();
options.TestExecutionMode = testExecutionMode;
options.DebugEnabled = m_debugEnabled;
stopWatch.Start();
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
try
{
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token
};
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Find the first test context with a web server configuration and use it
var webServerHost = SetupWebServerHost(testContexts, options);
ActiveWebServerHost = webServerHost;
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, testFileSummaries, overallSummary, webServerHost);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
catch (Exception e)
{
callback.ExceptionThrown(e);
ChutzpahTracer.TraceError(e, "Unhandled exception during Chutzpah test run");
return(overallSummary);
}
finally
{
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
}
}
private bool PerformBatchCompile(ITestMethodRunnerCallback callback, IEnumerable<TestContext> testContexts)
{
try
{
batchCompilerService.Compile(testContexts);
}
catch (ChutzpahCompilationFailedException e)
{
callback.ExceptionThrown(e, e.SettingsFile);
ChutzpahTracer.TraceError(e, "Error during batch compile from {0}", e.SettingsFile);
return false;
}
return true;
}
/// <inheritdoc/>
public void ExceptionThrown(string assemblyFilename, Exception exception)
{
callback.ExceptionThrown(testClass.TestAssembly, exception);
}
public void Compile(IEnumerable <TestContext> testContexts, ITestMethodRunnerCallback callback = null)
{
// Group the test contexts by test settings to run batch aware settings like compile
// For each test settings file that defines a compile step we will run it and update
// testContexts reference files accordingly.
var groupedTestContexts = testContexts.GroupBy(x => x.TestFileSettings);
foreach (var contextGroup in groupedTestContexts)
{
var testSettings = contextGroup.Key;
// If there is no compile setting then nothing to do here
if (testSettings.Compile == null)
{
continue;
}
// Build the mapping from source to output files and gather properties about them
var filePropeties = (
from file in contextGroup.SelectMany(x => x.ReferencedFiles).Where(x => !x.IsBuiltInDependency).Distinct()
where testSettings.Compile.Extensions.Any(x => file.Path.EndsWith(x, StringComparison.OrdinalIgnoreCase))
let sourceProperties = GetFileProperties(file.Path)
let sourceHasOutput = !testSettings.Compile.ExtensionsWithNoOutput.Any(x => file.Path.EndsWith(x, StringComparison.OrdinalIgnoreCase))
let outputPath = GetOutputPath(file.Path, testSettings.Compile)
let outputProperties = sourceHasOutput ? GetFileProperties(outputPath) : null
select new SourceCompileInfo {
SourceProperties = sourceProperties, OutputProperties = outputProperties, SourceHasOutput = sourceHasOutput
}).ToList();
var outputPathMap = filePropeties
.Where(x => x.SourceHasOutput)
.ToDictionary(x => x.SourceProperties.Path, x => x.OutputProperties.Path, StringComparer.OrdinalIgnoreCase);
// Check if the batch compile is needed
var shouldCompile = CheckIfCompileIsNeeded(testSettings, filePropeties);
// Run the batch compile if necessary
if (shouldCompile)
{
if (testSettings.Compile.Mode == BatchCompileMode.Executable)
{
RunBatchCompile(testSettings);
}
else
{
ChutzpahTracer.TraceWarning("Chutzpah determined generated .js files are missing but the compile mode is External so Chutzpah can't compile them. Test results may be wrong.");
}
}
else
{
ChutzpahTracer.TraceInformation("Skipping batch compile since all files are update to date for {0}", testSettings.SettingsFileName);
}
// Now that compile finished set generated path on all files who match the compiled extensions
var filesToUpdate = contextGroup.SelectMany(x => x.ReferencedFiles)
.Where(x => outputPathMap.ContainsKey(x.Path));
foreach (var file in filesToUpdate)
{
var outputPath = outputPathMap[file.Path];
if (outputPath != null && fileSystem.FileExists(outputPath))
{
file.GeneratedFilePath = outputPath;
ChutzpahTracer.TraceInformation("Found generated path for {0} at {1}", file.Path, outputPath);
}
else
{
// If we could not find the file at the configured path attempt to see if it co-located
ChutzpahTracer.TraceInformation("Unable to find generated path at configured location so attempting to see if generated file is co-located.");
var coLocatedOutputPath = Path.ChangeExtension(file.Path, ".js");
if (fileSystem.FileExists(coLocatedOutputPath))
{
file.GeneratedFilePath = coLocatedOutputPath;
ChutzpahTracer.TraceInformation("Found generated path for {0} at {1}", file.Path, coLocatedOutputPath);
}
else
{
var error = string.Format("Couldn't find generated path for {0} at {1} or at {2}", file.Path, outputPath, coLocatedOutputPath);
ChutzpahTracer.TraceError(error);
if (!testSettings.Compile.IgnoreMissingFiles.GetValueOrDefault())
{
// Throw and fail here since if we cant find the file we cannot be sure anything will run
var exception = new FileNotFoundException(error, outputPath);
callback.ExceptionThrown(exception, outputPath);
}
}
}
if (!string.IsNullOrWhiteSpace(file.GeneratedFilePath))
{
file.SourceMapFilePath = testSettings.Compile.UseSourceMaps.GetValueOrDefault() ? sourceMapDiscoverer.FindSourceMap(file.GeneratedFilePath) : null;
}
}
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
TestOptions options,
TestRunnerMode testRunnerMode,
ITestMethodRunnerCallback callback)
{
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
throw new ArgumentNullException("testPaths");
if (headlessBrowserPath == null)
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
var overallSummary = new TestCaseSummary();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token };
Parallel.ForEach(fileProbe.FindScriptFiles(testPaths, options.TestingMode), parallelOptions, testFile =>
{
try
{
if (cancellationSource.IsCancellationRequested) return;
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFile, options, out testContext))
{
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation("Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation("Invoking test runner on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.InputTestFile);
var testSummary = InvokeTestRunner(headlessBrowserPath,
options,
testContext,
testRunnerMode,
callback);
testFileSummaries.Enqueue(testSummary);
}
if (!m_debugEnabled && !options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation("Cleaning up test context artifacts");
// Don't clean up context if in debug mode
testContextBuilder.CleanupContext(testContext);
}
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
cancellationSource.Cancel();
}
}
catch (Exception e)
{
callback.ExceptionThrown(e, testFile.FullPath);
}
});
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
compilerCache.Save();
return overallSummary;
}
public void ExceptionThrown(TestAssembly testAssembly, Exception exception)
{
innerCallback.ExceptionThrown(testAssembly, exception);
}
