private IList <TestFileSummary> InvokeTestRunner(TestOptions options,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
var browser = (options.Engine ?? testContext.TestFileSettings.Engine).GetValueOrDefault();
var provider = testExecutionProviders.FirstOrDefault(x => x.CanHandleBrowser(browser));
if (provider == null)
{
throw new ArgumentException("Could not find browser");
}
provider.SetupEnvironment(options, testContext);
return(provider.Execute(options, testContext, testExecutionMode, callback));
}
private TestFileSummary InvokeTestRunner(string headlessBrowserPath,
TestOptions options,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext.TestHarnessPath, testContext.IsRemoteHarness);
string runnerArgs = BuildRunnerArgs(options, testContext, fileUrl, runnerPath, testExecutionMode);
Func <ProcessStream, TestFileSummary> streamProcessor =
processStream => testCaseStreamReaderFactory.Create().Read(processStream, options, testContext, callback, m_debugEnabled);
var processResult = process.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor);
HandleTestProcessExitCode(processResult.ExitCode, testContext.InputTestFile, processResult.Model.Errors, callback);
return(processResult.Model);
}
public IList <TestFileSummary> Execute(TestOptions testOptions,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext);
string runnerArgs = BuildRunnerArgs(testOptions, testContext, fileUrl, runnerPath, testExecutionMode);
var streamTimeout = ((testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500).GetValueOrDefault(); // Add buffer to timeout to account for serialization
Func <ProcessStreamStringSource, TestCaseStreamReadResult> streamProcessor =
processStream => readerFactory.Create().Read(processStream, testOptions, testContext, callback);
var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor, streamTimeout, null);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.TestFileSummaries.Select(x => x.Errors).FirstOrDefault(), callback);
return(processResult.Model.TestFileSummaries);
}
public IList <TestFileSummary> Execute(TestOptions testOptions,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext);
bool preventDownloadOfEngineDepenedencies = testContext.TestFileSettings.EngineOptions != null && testContext.TestFileSettings.EngineOptions.PreventDownloadOfEngineDepenedencies;
string runnerArgs = BuildRunnerArgs(testOptions, testContext, fileUrl, runnerPath, testExecutionMode, isRunningElevated, preventDownloadOfEngineDepenedencies);
var streamTimeout = ((testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500).GetValueOrDefault(); // Add buffer to timeout to account for serialization
TestCaseStreamReadResult streamProcessor(ProcessStreamStringSource processStream) => readerFactory.Create().Read(processStream, testOptions, testContext, callback);
var environmentVariables = BuildEnvironmentVariables();
var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor, streamTimeout, environmentVariables);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.TestFileSummaries.Select(x => x.Errors).FirstOrDefault(), callback);
return(processResult.Model.TestFileSummaries);
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
var testSummary = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testContext.InputTestFile,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
testFileSummaries.Enqueue(testSummary);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFile,
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile);
}
}
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, options.MaxDegreeOfParallelism);
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token
};
var scriptPaths = FindTestFiles(testPaths, options);
// Build test contexts in parallel
BuildTestContexts(options, scriptPaths, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
compilerCache.Save();
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3} {4} {5}",
runnerPath,
fileUrl,
testModeStr,
timeout,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
context.TestFileSettings.UserAgent);
return(runnerArgs);
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
}
catch (IOException)
{
// Mark this creation failed so we do not try to clean it up later
// This is to work around a bug in TestExplorer that runs chutzpah in parallel on
// the same files
// TODO(mmanela): Re-evalute if this is needed once they fix that bug
testContext.TestHarnessCreationFailed = true;
ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
throw;
}
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary <string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled &&
!testContext.TestHarnessCreationFailed &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token
};
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
private IList<TestFileSummary> InvokeTestRunner(string headlessBrowserPath,
TestOptions options,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext.TestHarnessPath, testContext.IsRemoteHarness);
string runnerArgs = BuildRunnerArgs(options, testContext, fileUrl, runnerPath, testExecutionMode);
Func<ProcessStream, IList<TestFileSummary>> streamProcessor =
processStream => testCaseStreamReaderFactory.Create().Read(processStream, options, testContext, callback, m_debugEnabled);
var processResult = process.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.Select(x => x.Errors).FirstOrDefault(), callback);
return processResult.Model;
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag<TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue<TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary<string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled
&& options.TestLaunchMode != TestLaunchMode.FullBrowser
&& options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
throw new ArgumentNullException("testPaths");
if (headlessBrowserPath == null)
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag<TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token };
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return overallSummary;
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return overallSummary;
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode, bool isRunningElevated, string chromeBrowserPath)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
runnerArgs = string.Format("\"{0}\" {1} {2} {3} {4} {5} {6} {7}",
runnerPath,
fileUrl,
testModeStr,
timeout,
isRunningElevated,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
$"\"{chromeBrowserPath}\"",
context.TestFileSettings.UserAgent);
return(runnerArgs);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode, bool isRunningElevated, string chromeBrowserPath)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
string inspectBrkArg = context.TestFileSettings.EngineOptions != null && context.TestFileSettings.EngineOptions.NodeInspect ? "--inspect-brk" : "";
var engineBrowserOptions = string.Empty;
if (context.TestFileSettings.BrowserArguments != null && options.Engine != null)
{
var matchingEntries = context.TestFileSettings.BrowserArguments.Where(x => x.Key.Equals(options.Engine.ToString(), StringComparison.OrdinalIgnoreCase));
if (matchingEntries.Any())
{
engineBrowserOptions = matchingEntries.First().Value;
}
}
runnerArgs = string.Format("{0} \"{1}\" {2} {3} {4} {5} {6} {7} \"{8}\" \"{9}\"",
inspectBrkArg,
runnerPath,
fileUrl,
testModeStr,
timeout,
isRunningElevated,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
$"\"{chromeBrowserPath}\"",
context.TestFileSettings.UserAgent,
engineBrowserOptions);
return(runnerArgs);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
var proxy = options.Proxy ?? context.TestFileSettings.Proxy;
var proxySetting = string.IsNullOrEmpty(proxy) ? "--proxy-type=none" : string.Format("--proxy={0}", proxy);
runnerArgs = string.Format("--ignore-ssl-errors=true {0} --ssl-protocol=any \"{1}\" {2} {3} {4} {5} {6}",
proxySetting,
runnerPath,
fileUrl,
testModeStr,
timeout,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
context.TestFileSettings.UserAgent);
return(runnerArgs);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3} {4} {5}",
runnerPath,
fileUrl,
testModeStr,
timeout,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
context.TestFileSettings.UserAgent);
return runnerArgs;
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds;
if (timeout.HasValue && timeout > 0)
{
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3}",
runnerPath,
fileUrl,
testModeStr,
timeout);
}
else
{
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2}", runnerPath, fileUrl, testModeStr);
}
return(runnerArgs);
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag<TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue<TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
