private static TestCaseSummary BuildTestCaseSummary()
{
var summary = new TestCaseSummary();
var fileSummary = new TestFileSummary("path1"){ TimeTaken = 1500};
fileSummary.AddTestCase(new TestCase
{
ModuleName = "module1",
TestName = "test1",
TestResults = new List<TestResult> { new TestResult { Passed = false, Message = "some failure" } }
});
fileSummary.AddTestCase(new TestCase
{
ModuleName = "module1",
TestName = "test2",
TestResults = new List<TestResult> { new TestResult { Passed = true } }
});
var fileSummary2 = new TestFileSummary("path>2") { TimeTaken = 2000 };
fileSummary2.AddTestCase(new TestCase
{
TestName = "test3",
TestResults = new List<TestResult> { new TestResult { Passed = true } }
});
fileSummary2.AddTestCase(new TestCase
{
TestName = "test<4",
TestResults = new List<TestResult> { new TestResult { Passed = false, Message = "bad<failure" } }
});
summary.Append(fileSummary);
summary.Append(fileSummary2);
return summary;
}
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
throw new ArgumentNullException("testPaths");
if (headlessBrowserPath == null)
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag<TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token };
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return overallSummary;
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return overallSummary;
}
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
TestOptions options,
TestRunnerMode testRunnerMode,
ITestMethodRunnerCallback callback)
{
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
throw new ArgumentNullException("testPaths");
if (headlessBrowserPath == null)
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
var overallSummary = new TestCaseSummary();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token };
Parallel.ForEach(fileProbe.FindScriptFiles(testPaths, options.TestingMode), parallelOptions, testFile =>
{
try
{
if (cancellationSource.IsCancellationRequested) return;
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFile, options, out testContext))
{
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation("Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation("Invoking test runner on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.InputTestFile);
var testSummary = InvokeTestRunner(headlessBrowserPath,
options,
testContext,
testRunnerMode,
callback);
testFileSummaries.Enqueue(testSummary);
}
if (!m_debugEnabled && !options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation("Cleaning up test context artifacts");
// Don't clean up context if in debug mode
testContextBuilder.CleanupContext(testContext);
}
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
cancellationSource.Cancel();
}
}
catch (Exception e)
{
callback.ExceptionThrown(e, testFile.FullPath);
}
});
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
compilerCache.Save();
return overallSummary;
}