private void RunTests(IEnumerable <string> filePaths, bool openInBrowser = false, bool withCodeCoverage = false, bool withDebugger = false)
{
if (!testingInProgress)
{
lock (syncLock)
{
if (!testingInProgress)
{
dte.Documents.SaveAll();
var solutionDir = Path.GetDirectoryName(dte.Solution.FullName);
testingInProgress = true;
Task.Factory.StartNew(
() =>
{
try
{
// If setttings file environments have not yet been initialized, do so here.
if (settingsEnvironments == null || settingsEnvironments.Count == 0)
{
InitializeSettingsFileEnvironments();
}
var options = new TestOptions
{
MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism,
CoverageOptions = new CoverageOptions
{
Enabled = withCodeCoverage
},
CustomTestLauncher = withDebugger ? new VsDebuggerTestLauncher() : null,
TestLaunchMode = GetTestLaunchMode(openInBrowser, withDebugger),
ChutzpahSettingsFileEnvironments = settingsEnvironments
};
var result = testRunner.RunTests(filePaths, options, runnerCallback);
if (result.CoverageObject != null)
{
var path = CoverageOutputGenerator.WriteHtmlFile(Path.Combine(solutionDir, Constants.CoverageHtmlFileName), result.CoverageObject);
processHelper.LaunchFileInBrowser(path);
}
}
catch (Exception e)
{
Logger.Log("Error while running tests", "ChutzpahPackage", e);
}
finally
{
testingInProgress = false;
}
});
}
}
}
}
private void RunTests(IEnumerable <string> filePaths, bool withCodeCoverage, bool openInBrowser)
{
if (!testingInProgress)
{
lock (syncLock)
{
if (!testingInProgress)
{
dte.Documents.SaveAll();
var solutionDir = Path.GetDirectoryName(dte.Solution.FullName);
testingInProgress = true;
Task.Factory.StartNew(
() =>
{
try
{
var options = new TestOptions
{
TestFileTimeoutMilliseconds = Settings.TimeoutMilliseconds,
MaxDegreeOfParallelism = Settings.MaxDegreeOfParallelism,
CoverageOptions = new CoverageOptions
{
Enabled = withCodeCoverage
},
OpenInBrowser = openInBrowser
};
var result = testRunner.RunTests(filePaths, options, runnerCallback);
if (result.CoverageObject != null)
{
var path = CoverageOutputGenerator.WriteHtmlFile(solutionDir, result.CoverageObject);
processHelper.LaunchFileInBrowser(path);
}
}
catch (Exception e)
{
Logger.Log("Error while running tests", "ChutzpahPackage", e);
}
finally
{
testingInProgress = false;
}
});
}
}
}
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
var testSummary = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testContext.InputTestFile,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
testFileSummaries.Enqueue(testSummary);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFile,
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile);
}
}
}
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
}
catch (IOException)
{
// Mark this creation failed so we do not try to clean it up later
// This is to work around a bug in TestExplorer that runs chutzpah in parallel on
// the same files
// TODO(mmanela): Re-evalute if this is needed once they fix that bug
testContext.TestHarnessCreationFailed = true;
ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
throw;
}
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary <string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled &&
!testContext.TestHarnessCreationFailed &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
