public void FileError(TestError error)
{
lock (sync)
{
nestedCallback.FileError(error);
}
}
public override void FileError(TestError error)
{
ClearCounter();
Console.ForegroundColor = ConsoleColor.Red;
var errorMessage = GetFileErrorMessage(error);
Console.WriteLine(errorMessage);
Console.ResetColor();
}
public static string FormatFileErrorMessage(TestError error)
{
var stack = "";
foreach (var item in error.Stack)
{
if (!string.IsNullOrEmpty(item.Function))
{
stack += "at " + item.Function + " ";
}
if (!string.IsNullOrEmpty(item.File))
{
stack += "in " + item.File;
}
if (!string.IsNullOrEmpty(item.Line))
{
stack += string.Format(" (line {0})", item.Line);
}
stack += "\n";
}
return string.Format("Error: {0}\n{1}While Running:{2}\n", error.Message, stack, error.InputTestFile);
}
public override void FileError(TestError error)
{
var output = string.Concat("Error in test file : ", error.InputTestFile, "\r\n", error.Message, "\r\n", error.Stack);
Errors.Add(output);
}
private static void HandleTestProcessExitCode(int exitCode, string inputTestFile, IList<TestError> errors, ITestMethodRunnerCallback callback)
{
string errorMessage = null;
switch ((TestProcessExitCode)exitCode)
{
case TestProcessExitCode.AllPassed:
case TestProcessExitCode.SomeFailed:
return;
case TestProcessExitCode.Timeout:
errorMessage = "Timeout occurred when executing test file";
break;
default:
errorMessage = "Unknown error occurred when executing test file. Received exit code of " + exitCode;
break;
}
if (!string.IsNullOrEmpty(errorMessage))
{
var error = new TestError
{
InputTestFile = inputTestFile,
Message = errorMessage
};
errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError("Headless browser returned with an error: {0}", errorMessage);
}
}
private void BuildTestContexts(
TestOptions options,
List<List<PathInfo>> scriptPathGroups,
ParallelOptions parallelOptions,
CancellationTokenSource cancellationSource,
int resultCount,
ConcurrentBag<TestContext> testContexts,
ITestMethodRunnerCallback callback,
TestCaseSummary overallSummary)
{
Parallel.ForEach(scriptPathGroups, parallelOptions, testFiles =>
{
var pathString = string.Join(",", testFiles.Select(x => x.FullPath));
ChutzpahTracer.TraceInformation("Building test context for {0}", pathString);
try
{
if (cancellationSource.IsCancellationRequested) return;
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFiles, options, out testContext))
{
testContexts.Add(testContext);
}
else
{
ChutzpahTracer.TraceWarning("Unable to build test context for {0}", pathString);
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
ChutzpahTracer.TraceError("File search limit hit!!!");
cancellationSource.Cancel();
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testFiles.Select(x => x.FullPath).FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during building test context for {0}", pathString);
}
finally
{
ChutzpahTracer.TraceInformation("Finished building test context for {0}", pathString);
}
});
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag<TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue<TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary<string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled
&& options.TestLaunchMode != TestLaunchMode.FullBrowser
&& options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
protected virtual string GetFileErrorMessage(TestError error)
{
return FormatFileErrorMessage(error);
}
public static string FormatFileErrorMessage(TestError error)
{
var stack = error.GetFormattedStackTrace();
return string.Format("Error: {0}\n{1}While Running:{2}\n", error.Message, stack, error.InputTestFile);
}
public virtual void FileError(TestError error) { }
public override void FileError(TestError error)
{
var errorMessage = GetFileErrorMessage(error);
Console.Write(errorMessage);
}
public override void FileError(TestError error)
{
testPane.OutputString(GetFileErrorMessage(error));
}
public override void FileError(TestError error)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, GetFileErrorMessage(error));
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag<TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue<TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}