private IChutzpahWebServerHost SetupWebServerHost(ConcurrentBag <TestContext> testContexts, TestOptions options)
{
var needsServer = options.Engine.GetValueOrDefault() != Engine.Phantom;
IChutzpahWebServerHost webServerHost = null;
var contextUsingWebServer = testContexts.Where(x => x.TestFileSettings.Server?.Enabled.GetValueOrDefault() == true).ToList();
var contextWithChosenServerConfiguration = contextUsingWebServer.FirstOrDefault();
if (contextWithChosenServerConfiguration == null && needsServer)
{
foreach (var testContext in testContexts)
{
testContext.TestFileSettings.Server = ForcedChutzpahWebServerConfiguration.Instance;
}
contextUsingWebServer = testContexts.ToList();
contextWithChosenServerConfiguration = testContexts.FirstOrDefault();
}
if (contextWithChosenServerConfiguration != null)
{
var webServerConfiguration = contextWithChosenServerConfiguration.TestFileSettings.Server;
webServerHost = webServerFactory.CreateServer(webServerConfiguration, ActiveWebServerHost);
// Stash host object on context for use in url generation
contextUsingWebServer.ForEach(x => x.WebServerHost = webServerHost);
}
return(webServerHost);
}
private IChutzpahWebServerHost SetupWebServerHost(ConcurrentBag <TestContext> testContexts, IChutzpahWebServerHost activeWebServerHost)
{
IChutzpahWebServerHost webServerHost = null;
var contextUsingWebServer = testContexts.Where(x => x.TestFileSettings.Server != null && x.TestFileSettings.Server.Enabled.GetValueOrDefault()).ToList();
var contextWithChosenServerConfiguration = contextUsingWebServer.FirstOrDefault();
if (contextWithChosenServerConfiguration != null)
{
var webServerConfiguration = contextWithChosenServerConfiguration.TestFileSettings.Server;
webServerHost = webServerFactory.CreateServer(webServerConfiguration, ActiveWebServerHost);
// Stash host object on context for use in url generation
contextUsingWebServer.ForEach(x => x.WebServerHost = webServerHost);
}
return(webServerHost);
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary,
IChutzpahWebServerHost webServerHost)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
testContext.TaskId = Task.CurrentId.GetValueOrDefault();
try
{
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
}
catch (IOException)
{
// Mark this creation failed so we do not try to clean it up later
// This is to work around a bug in TestExplorer that runs chutzpah in parallel on
// the same files
// TODO(mmanela): Re-evalute if this is needed once they fix that bug
testContext.TestHarnessCreationFailed = true;
ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
throw;
}
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.OpenInBrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.OpenInBrowserName);
browserArgs = new Dictionary <string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.OpenInBrowserArgs }
};
}
process.LaunchFileInBrowser(testContext, testContext.TestHarnessPath, options.OpenInBrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
callback.TestContextStarted(testContext);
var testSummaries = InvokeTestRunner(
options,
testContext,
testExecutionMode,
callback);
callback.TestContextFinished(testContext);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(testContext, error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled &&
!testContext.TestHarnessCreationFailed &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
if (webServerHost != null &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
webServerHost.Dispose();
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
IChutzpahWebServerHost activeWebServerHost = null)
{
var overallSummary = new TestCaseSummary();
options.TestExecutionMode = testExecutionMode;
options.DebugEnabled = m_debugEnabled;
stopWatch.Start();
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
try
{
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token
};
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Find the first test context with a web server configuration and use it
var webServerHost = SetupWebServerHost(testContexts, options);
ActiveWebServerHost = webServerHost;
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, testFileSummaries, overallSummary, webServerHost);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
catch (Exception e)
{
callback.ExceptionThrown(e);
ChutzpahTracer.TraceError(e, "Unhandled exception during Chutzpah test run");
return(overallSummary);
}
finally
{
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
}
}
public IChutzpahWebServerHost CreateServer(ChutzpahWebServerConfiguration configuration, IChutzpahWebServerHost activeWebServerHost)
{
if (activeWebServerHost != null &&
activeWebServerHost.IsRunning &&
activeWebServerHost.RootPath.Equals(configuration.RootPath, StringComparison.OrdinalIgnoreCase))
{
// If the requested server is already running just re-use it
return(activeWebServerHost);
}
var builtInDependencyFolder = fileProbe.BuiltInDependencyDirectory;
return(BuildHost(configuration.RootPath, configuration.DefaultPort.Value, builtInDependencyFolder));
}
