public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
{
if (processStream == null) throw new ArgumentNullException("processStream");
if (testOptions == null) throw new ArgumentNullException("testOptions");
if (testContext == null) throw new ArgumentNullException("testContext");
lastTestEvent = DateTime.Now;
var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization
var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, callback, debugEnabled));
while (readerTask.Status == TaskStatus.WaitingToRun
|| (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout))
{
Thread.Sleep(100);
}
if (readerTask.IsCompleted)
{
ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile);
return readerTask.Result;
}
else
{
// We timed out so kill the process and return an empty test file summary
ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds);
processStream.TimedOut = true;
processStream.KillProcess();
return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList();
}
}
public TestHarness(ChutzpahTestSettingsFile chutzpahTestSettings, TestOptions testOptions, IEnumerable<ReferencedFile> referencedFiles, IFileSystemWrapper fileSystem)
{
this.chutzpahTestSettings = chutzpahTestSettings;
this.testOptions = testOptions;
this.referencedFiles = referencedFiles;
this.fileSystem = fileSystem;
BuildTags(referencedFiles);
CleanupTestHarness();
}
public TestContext BuildContext(string file, TestOptions options)
{
if (String.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
PathInfo pathInfo = fileProbe.GetPathInfo(file);
return BuildContext(pathInfo, options);
}
public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
{
if (processStream == null) throw new ArgumentNullException("processStream");
if (testOptions == null) throw new ArgumentNullException("testOptions");
if (testContext == null) throw new ArgumentNullException("testContext");
lastTestEvent = DateTime.Now;
var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization
var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled)
|| (testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);
var streamingTestFileContexts = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.Select(x => new StreamingTestFileContext(x, testContext, codeCoverageEnabled))
.ToList();
var deferredEvents = new List<Action<StreamingTestFileContext>>();
var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, streamingTestFileContexts, deferredEvents, callback, debugEnabled));
while (readerTask.Status == TaskStatus.WaitingToRun
|| (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout))
{
Thread.Sleep(100);
}
if (readerTask.IsCompleted)
{
ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile);
return readerTask.Result;
}
else
{
// Since we times out make sure we play the deferred events so we do not lose errors
// We will just attach these events to the first test context at this point since we do
// not know where they belong
PlayDeferredEvents(streamingTestFileContexts.FirstOrDefault(), deferredEvents);
// We timed out so kill the process and return an empty test file summary
ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds);
processStream.TimedOut = true;
processStream.KillProcess();
return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList();
}
}
public void CreateTestHarness(TestContext testContext, TestOptions options)
{
if (!string.IsNullOrEmpty(testContext.TestHarnessPath))
{
// If we already have a test harness path then this means we executed on an existing html file or url
// So we dont need to generate the harness
return;
}
SetupAmdPathsIfNeeded(testContext.TestFileSettings, testContext.ReferencedFiles.ToList(), testContext.TestHarnessDirectory);
string testFilePathHash = hasher.Hash(string.Join(",",testContext.InputTestFiles));
string testHtmlFilePath = Path.Combine(testContext.TestHarnessDirectory, string.Format(Constants.ChutzpahTemporaryFileFormat, testFilePathHash, "test.html"));
testContext.TemporaryFiles.Add(testHtmlFilePath);
var templatePath = GetTestHarnessTemplatePath(testContext.FrameworkDefinition, testContext.TestFileSettings);
string testHtmlTemplate = fileSystem.GetText(templatePath);
var harness = new TestHarness(testContext.TestFileSettings, options, testContext.ReferencedFiles, fileSystem);
if (testContext.CoverageEngine != null)
{
testContext.CoverageEngine.PrepareTestHarnessForCoverage(harness, testContext.FrameworkDefinition, testContext.TestFileSettings);
}
var kvps = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.SelectMany(x => x.FrameworkReplacements);
var frameworkReplacements = new Dictionary<string, string>();
foreach (var pair in kvps)
{
frameworkReplacements[pair.Key] = pair.Value;
}
string testHtmlText = harness.CreateHtmlText(testHtmlTemplate, frameworkReplacements);
fileSystem.Save(testHtmlFilePath, testHtmlText);
testContext.TestHarnessPath = testHtmlFilePath;
}
private ICoverageEngine GetConfiguredCoverageEngine(TestOptions options, ChutzpahTestSettingsFile chutzpahTestSettings)
{
// Don't run code coverage if in discovery mode
if (options.TestExecutionMode == TestExecutionMode.Discovery)
{
return(null);
}
var codeCoverageEnabled = options.CoverageOptions.ShouldRunCoverage(chutzpahTestSettings.CodeCoverageExecutionMode);
if (!codeCoverageEnabled)
{
return(null);
}
ChutzpahTracer.TraceInformation("Setting up code coverage in test context");
var coverageEngine = coverageEngineFactory.CreateCoverageEngine();
coverageEngine.ClearPatterns();
coverageEngine.AddIncludePatterns(chutzpahTestSettings.CodeCoverageIncludes.Concat(options.CoverageOptions.IncludePatterns));
coverageEngine.AddExcludePatterns(chutzpahTestSettings.CodeCoverageExcludes.Concat(options.CoverageOptions.ExcludePatterns));
coverageEngine.AddIgnorePatterns(chutzpahTestSettings.CodeCoverageIgnores.Concat(options.CoverageOptions.IgnorePatterns));
return(coverageEngine);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3} {4} {5}",
runnerPath,
fileUrl,
testModeStr,
timeout,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
context.TestFileSettings.UserAgent);
return runnerArgs;
}
static int RunTests(CommandLine commandLine)
{
var testRunner = TestRunner.Create(debugEnabled: commandLine.Debug);
if (commandLine.Trace)
{
ChutzpahTracer.AddFileListener();
}
Console.WriteLine();
TestCaseSummary testResultsSummary = null;
try
{
var callback = commandLine.TeamCity
? (ITestMethodRunnerCallback)new TeamCityConsoleRunnerCallback()
: new StandardConsoleRunnerCallback(commandLine.Silent, commandLine.VsOutput, commandLine.ShowFailureReport, commandLine.FailOnError);
callback = new ParallelRunnerCallbackAdapter(callback);
var testOptions = new TestOptions
{
TestLaunchMode = commandLine.OpenInBrowser ? TestLaunchMode.FullBrowser : TestLaunchMode.HeadlessBrowser,
BrowserName = commandLine.BrowserName,
TestFileTimeoutMilliseconds = commandLine.TimeOutMilliseconds,
MaxDegreeOfParallelism = commandLine.Parallelism,
ChutzpahSettingsFileEnvironments = commandLine.SettingsFileEnvironments,
CoverageOptions = new CoverageOptions
{
Enabled = commandLine.Coverage,
IncludePatterns = (commandLine.CoverageIncludePatterns ?? "").Split(new[]{','},StringSplitOptions.RemoveEmptyEntries),
ExcludePatterns = (commandLine.CoverageExcludePatterns ?? "").Split(new[]{','},StringSplitOptions.RemoveEmptyEntries),
IgnorePatterns = (commandLine.CoverageIgnorePatterns ?? "").Split(new[]{','},StringSplitOptions.RemoveEmptyEntries)
}
};
if (!commandLine.Discovery)
{
testResultsSummary = testRunner.RunTests(commandLine.Files, testOptions, callback);
ProcessTestSummaryTransformers(commandLine, testResultsSummary);
}
else
{
Console.WriteLine("Test Discovery");
var tests = testRunner.DiscoverTests(commandLine.Files, testOptions).ToList();
Console.WriteLine("\nDiscovered {0} tests", tests.Count);
foreach (var test in tests)
{
Console.WriteLine("Test '{0}:{1}' from '{2}'", test.ModuleName, test.TestName, test.InputTestFile);
}
return 0;
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
var failedCount = testResultsSummary.FailedCount;
if (commandLine.FailOnError && testResultsSummary.Errors.Any())
{
return failedCount > 0 ? failedCount : 1;
}
return failedCount;
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag<TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue<TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary<string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled
&& options.TestLaunchMode != TestLaunchMode.FullBrowser
&& options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
private IEnumerable<PathInfo> FindTestFiles(IEnumerable<string> testPaths, TestOptions options)
{
IEnumerable<PathInfo> scriptPaths = Enumerable.Empty<PathInfo>();
// If the path list contains only chutzpah.json files then use those files for getting the list of test paths
var testPathList = testPaths.Distinct(StringComparer.OrdinalIgnoreCase).ToList();
if (testPathList.All(testPath => Path.GetFileName(testPath).Equals(Constants.SettingsFileName, StringComparison.OrdinalIgnoreCase)))
{
ChutzpahTracer.TraceInformation("Using Chutzpah.json files to find tests");
foreach (var path in testPathList)
{
var chutzpahJsonPath = fileProbe.FindFilePath(path);
if (chutzpahJsonPath == null)
{
ChutzpahTracer.TraceWarning("Supplied chutzpah.json path {0} does not exist", path);
}
var settingsFile = testSettingsService.FindSettingsFile(chutzpahJsonPath, options.ChutzpahSettingsFileEnvironments);
var pathInfos = fileProbe.FindScriptFiles(settingsFile);
scriptPaths = scriptPaths.Concat(pathInfos);
}
}
else
{
scriptPaths = fileProbe.FindScriptFiles(testPathList);
}
return scriptPaths
.Where(x => x.FullPath != null)
.ToList(); ;
}
public TestContext BuildContext(IEnumerable <PathInfo> files, TestOptions options)
{
if (files == null)
{
throw new ArgumentNullException("testFilePathInfo");
}
if (!files.Any())
{
ChutzpahTracer.TraceInformation("No files given to build test context for");
return(null);
}
var fileCount = files.Count();
var allFilePathString = string.Join(",", files.Select(x => x.FullPath));
ChutzpahTracer.TraceInformation("Building test context for {0}", allFilePathString);
// Make sure all test paths have been resolved to real files
var missingPaths = files.Where(x => x.FullPath == null).ToList();
if (missingPaths.Any())
{
throw new FileNotFoundException("Unable to find files: " + string.Join(",", missingPaths.Select(x => x.Path)));
}
// Make sure all test paths have a valid file type
if (!files.Select(x => x.Type).All(IsValidTestPathType))
{
throw new ArgumentException("Expecting valid file or a url");
}
if (fileCount > 1 && files.Any(x => x.Type == PathType.Url || x.Type == PathType.Html))
{
throw new InvalidOperationException("Cannot build a batch context for Url or Html test files");
}
// We use the first file's directory to find the chutzpah.json file and to test framework
// since we assume all files in the batch must have the same values for those
PathType firstFileKind = files.First().Type;
string firstFilePath = files.First().FullPath;
var chutzpahTestSettings = settingsService.FindSettingsFile(firstFilePath, options.ChutzpahSettingsFileEnvironments);
// Exclude any files that are not included based on the settings file
SettingsFileTestPath matchingTestSettingsPath;
var testedPaths = files.Select(f => new PathWithTestSetting {
File = f,
IsIncluded = IsTestPathIncluded(f.FullPath, chutzpahTestSettings, out matchingTestSettingsPath),
MatchingTestSetting = matchingTestSettingsPath
}).ToList();
if (testedPaths.Any(x => !x.IsIncluded))
{
var pathString = string.Join(",", testedPaths.Where(x => !x.IsIncluded).Select(x => x.File.FullPath));
ChutzpahTracer.TraceInformation("Excluding test files {0} given chutzpah.json settings", pathString);
testedPaths = testedPaths.Where(x => x.IsIncluded).ToList();
if (!testedPaths.Any())
{
return(null);
}
}
IFrameworkDefinition definition;
if (TryDetectFramework(testedPaths.First().File, chutzpahTestSettings, out definition))
{
// For HTML test files we don't need to create a test harness to just return this file
if (firstFileKind == PathType.Html || firstFileKind == PathType.Url)
{
ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", firstFileKind);
return(new TestContext
{
ReferencedFiles = new List <ReferencedFile> {
new ReferencedFile {
IsFileUnderTest = true, Path = firstFilePath
}
},
InputTestFiles = new[] { firstFilePath },
FirstInputTestFile = firstFilePath,
InputTestFilesString = firstFilePath,
TestHarnessPath = firstFilePath,
IsRemoteHarness = firstFileKind == PathType.Url,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
});
}
var temporaryFiles = new List <string>();
string firstInputTestFileDir = Path.GetDirectoryName(firstFilePath);
var testHarnessDirectory = GetTestHarnessDirectory(chutzpahTestSettings, firstInputTestFileDir);
var referencedFiles = GetFilesUnderTest(testedPaths, chutzpahTestSettings).ToList();
referenceProcessor.GetReferencedFiles(referencedFiles, definition, chutzpahTestSettings);
IEnumerable <string> deps = definition.GetFileDependencies(chutzpahTestSettings);
var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles);
AddTestFrameworkDependencies(deps, referencedFiles);
var testFiles = referencedFiles.Where(x => x.IsFileUnderTest).Select(x => x.Path).ToList();
return(new TestContext
{
FrameworkDefinition = definition,
CoverageEngine = coverageEngine,
InputTestFiles = testFiles,
FirstInputTestFile = testFiles.FirstOrDefault(),
InputTestFilesString = string.Join(",", testFiles),
TestHarnessDirectory = testHarnessDirectory,
ReferencedFiles = referencedFiles,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
TemporaryFiles = temporaryFiles,
TestFileSettings = chutzpahTestSettings
});
}
else
{
ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", firstFilePath);
}
return(null);
}
private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
throw new ArgumentNullException("testPaths");
if (headlessBrowserPath == null)
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag<TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token };
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return overallSummary;
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return overallSummary;
}
private string CreateTestHarness(
IFrameworkDefinition definition,
ChutzpahTestSettingsFile chutzpahTestSettings,
TestOptions options,
string inputTestFilePath,
IEnumerable<ReferencedFile> referencedFiles,
ICoverageEngine coverageEngine,
IList<string> temporaryFiles)
{
string inputTestFileDir = Path.GetDirectoryName(inputTestFilePath);
string testFilePathHash = hasher.Hash(inputTestFilePath);
string testHarnessDirectory;
switch (chutzpahTestSettings.TestHarnessLocationMode)
{
case TestHarnessLocationMode.TestFileAdjacent:
testHarnessDirectory = inputTestFileDir;
break;
case TestHarnessLocationMode.SettingsFileAdjacent:
testHarnessDirectory = chutzpahTestSettings.SettingsFileDirectory;
break;
case TestHarnessLocationMode.Custom:
testHarnessDirectory = chutzpahTestSettings.TestHarnessDirectory;
break;
default:
throw new ArgumentOutOfRangeException("chutzpahTestSettings");
}
string testHtmlFilePath = Path.Combine(testHarnessDirectory, string.Format(Constants.ChutzpahTemporaryFileFormat, testFilePathHash, "test.html"));
temporaryFiles.Add(testHtmlFilePath);
string templatePath = fileProbe.GetPathInfo(Path.Combine(Constants.TestFileFolder, definition.TestHarness)).FullPath;
string testHtmlTemplate = fileSystem.GetText(templatePath);
string amdModulePath = "";
if (chutzpahTestSettings.TestHarnessReferenceMode == TestHarnessReferenceMode.AMD && inputTestFilePath.Contains(testHarnessDirectory))
{
amdModulePath = inputTestFilePath
.Replace(Path.GetExtension(inputTestFilePath), "")
.Replace(testHarnessDirectory, "")
.Replace("\\", "/")
.Trim('/', '\\');
}
var harness = new TestHarness(chutzpahTestSettings, options, referencedFiles, fileSystem, amdModulePath);
if (coverageEngine != null)
{
coverageEngine.PrepareTestHarnessForCoverage(harness, definition);
}
string testFileContents = fileSystem.GetText(inputTestFilePath);
var frameworkReplacements = definition.GetFrameworkReplacements(chutzpahTestSettings, inputTestFilePath, testFileContents)
?? new Dictionary<string, string>();
string testHtmlText = harness.CreateHtmlText(testHtmlTemplate, frameworkReplacements);
fileSystem.Save(testHtmlFilePath, testHtmlText);
return testHtmlFilePath;
}
public TestContext BuildContext(IEnumerable<string> files, TestOptions options)
{
if (files == null)
{
throw new ArgumentNullException("files");
}
return BuildContext(files.Select(fileProbe.GetPathInfo), options);
}
public TestContext BuildContext(IEnumerable<PathInfo> files, TestOptions options)
{
if (files == null)
{
throw new ArgumentNullException("testFilePathInfo");
}
if (!files.Any())
{
ChutzpahTracer.TraceInformation("No files given to build test context for");
return null;
}
var fileCount = files.Count();
var allFilePathString = string.Join(",", files.Select(x => x.FullPath));
ChutzpahTracer.TraceInformation("Building test context for '{0}'", allFilePathString);
// Make sure all test paths have been resolved to real files
var missingPaths = files.Where(x => x.FullPath == null).ToList();
if (missingPaths.Any())
{
throw new FileNotFoundException("Unable to find files: " + string.Join(",", missingPaths.Select(x => x.Path)));
}
// Make sure all test paths have a valid file type
if (!files.Select(x => x.Type).All(IsValidTestPathType))
{
throw new ArgumentException("Expecting valid file or a url");
}
if (fileCount > 1 && files.Any(x => x.Type == PathType.Url || x.Type == PathType.Html))
{
throw new InvalidOperationException("Cannot build a batch context for Url or Html test files");
}
// We use the first file's directory to find the chutzpah.json file and to test framework
// since we assume all files in the batch must have the same values for those
PathType firstFileKind = files.First().Type;
string firstFilePath = files.First().FullPath;
var chutzpahTestSettings = settingsService.FindSettingsFile(firstFilePath, options.ChutzpahSettingsFileEnvironments);
// Exclude any files that are not included based on the settings file
SettingsFileTestPath matchingTestSettingsPath;
var testedPaths = files.Select(f => new PathWithTestSetting {
File = f,
IsIncluded = IsTestPathIncluded(f.FullPath, chutzpahTestSettings, out matchingTestSettingsPath),
MatchingTestSetting = matchingTestSettingsPath }).ToList();
if (testedPaths.Any(x => !x.IsIncluded))
{
var pathString = string.Join(",", testedPaths.Where(x => !x.IsIncluded).Select(x => x.File.FullPath));
ChutzpahTracer.TraceInformation("Excluding test files {0} given chutzpah.json settings", pathString);
testedPaths = testedPaths.Where(x => x.IsIncluded).ToList();
if (!testedPaths.Any())
{
return null;
}
}
IFrameworkDefinition definition;
if (TryDetectFramework(testedPaths.First().File, chutzpahTestSettings, out definition))
{
// For HTML test files we don't need to create a test harness to just return this file
if (firstFileKind == PathType.Html || firstFileKind == PathType.Url)
{
ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", firstFileKind);
return new TestContext
{
ReferencedFiles = new List<ReferencedFile> { new ReferencedFile { IsFileUnderTest = true, Path = firstFilePath } },
InputTestFiles = new[] { firstFilePath },
FirstInputTestFile = firstFilePath,
InputTestFilesString = firstFilePath,
TestHarnessPath = firstFilePath,
IsRemoteHarness = firstFileKind == PathType.Url,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
};
}
var temporaryFiles = new List<string>();
string firstInputTestFileDir = Path.GetDirectoryName(firstFilePath);
var testHarnessDirectory = GetTestHarnessDirectory(chutzpahTestSettings, firstInputTestFileDir);
var referencedFiles = GetFilesUnderTest(testedPaths, chutzpahTestSettings).ToList();
referenceProcessor.GetReferencedFiles(referencedFiles, definition, chutzpahTestSettings);
IEnumerable<string> deps = definition.GetFileDependencies(chutzpahTestSettings);
var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles);
AddTestFrameworkDependencies(deps, referencedFiles);
var testFiles = referencedFiles.Where(x => x.IsFileUnderTest).Select(x => x.Path).ToList();
return new TestContext
{
FrameworkDefinition = definition,
CoverageEngine = coverageEngine,
InputTestFiles = testFiles,
FirstInputTestFile = testFiles.FirstOrDefault(),
InputTestFilesString = string.Join(",", testFiles),
TestHarnessDirectory = testHarnessDirectory,
ReferencedFiles = referencedFiles,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
TemporaryFiles = temporaryFiles,
TestFileSettings = chutzpahTestSettings
};
}
else
{
ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", firstFilePath);
}
return null;
}
public bool TryBuildContext(PathInfo file, TestOptions options, out TestContext context)
{
context = BuildContext(file, options);
return context != null;
}
public TestContext BuildContext(PathInfo file, TestOptions options)
{
return BuildContext(new List<PathInfo> { file }, options);
}
private ICoverageEngine GetConfiguredCoverageEngine(TestOptions options, ChutzpahTestSettingsFile chutzpahTestSettings)
{
// Don't run code coverage if in discovery mode
if (options.TestExecutionMode == TestExecutionMode.Discovery) return null;
var codeCoverageEnabled = options.CoverageOptions.ShouldRunCoverage(chutzpahTestSettings.CodeCoverageExecutionMode);
if (!codeCoverageEnabled) return null;
ChutzpahTracer.TraceInformation("Setting up code coverage in test context");
var coverageEngine = coverageEngineFactory.CreateCoverageEngine();
coverageEngine.ClearPatterns();
coverageEngine.AddIncludePatterns(chutzpahTestSettings.CodeCoverageIncludes.Concat(options.CoverageOptions.IncludePatterns));
coverageEngine.AddExcludePatterns(chutzpahTestSettings.CodeCoverageExcludes.Concat(options.CoverageOptions.ExcludePatterns));
coverageEngine.AddIgnorePatterns(chutzpahTestSettings.CodeCoverageIgnores.Concat(options.CoverageOptions.IgnorePatterns));
return coverageEngine;
}
public bool TryBuildContext(IEnumerable<string> files, TestOptions options, out TestContext context)
{
context = BuildContext(files, options);
return context != null;
}
private IChutzpahWebServerHost SetupWebServerHost(ConcurrentBag <TestContext> testContexts, TestOptions options)
{
var needsServer = options.Engine.GetValueOrDefault() != Engine.Phantom;
IChutzpahWebServerHost webServerHost = null;
var contextUsingWebServer = testContexts.Where(x => x.TestFileSettings.Server?.Enabled.GetValueOrDefault() == true).ToList();
var contextWithChosenServerConfiguration = contextUsingWebServer.FirstOrDefault();
if (contextWithChosenServerConfiguration == null && needsServer)
{
foreach (var testContext in testContexts)
{
testContext.TestFileSettings.Server = ForcedChutzpahWebServerConfiguration.Instance;
}
contextUsingWebServer = testContexts.ToList();
contextWithChosenServerConfiguration = testContexts.FirstOrDefault();
}
if (contextWithChosenServerConfiguration != null)
{
var webServerConfiguration = contextWithChosenServerConfiguration.TestFileSettings.Server;
webServerHost = webServerFactory.CreateServer(webServerConfiguration, ActiveWebServerHost);
// Stash host object on context for use in url generation
contextUsingWebServer.ForEach(x => x.WebServerHost = webServerHost);
}
return(webServerHost);
}
private ICoverageEngine GetConfiguredCoverageEngine(TestOptions options, ChutzpahTestSettingsFile chutzpahTestSettings)
{
if (options == null || !options.CoverageOptions.Enabled) return null;
ChutzpahTracer.TraceInformation("Setting up code coverage in test context");
mainCoverageEngine.IncludePatterns = chutzpahTestSettings.CodeCoverageIncludes.Concat(options.CoverageOptions.IncludePatterns).ToList();
mainCoverageEngine.ExcludePatterns = chutzpahTestSettings.CodeCoverageExcludes.Concat(options.CoverageOptions.ExcludePatterns).ToList();
return mainCoverageEngine;
}
static int RunTests(CommandLine commandLine)
{
var testRunner = TestRunner.Create(debugEnabled: commandLine.Debug);
if (commandLine.Trace)
{
ChutzpahTracer.AddFileListener();
}
var chutzpahAssemblyName = testRunner.GetType().Assembly.GetName();
Console.WriteLine();
Console.WriteLine("chutzpah.dll: Version {0}", chutzpahAssemblyName.Version);
Console.WriteLine();
TestCaseSummary testResultsSummary = null;
try
{
var callback = commandLine.TeamCity
? (ITestMethodRunnerCallback) new TeamCityConsoleRunnerCallback()
: new StandardConsoleRunnerCallback(commandLine.Silent, commandLine.VsOutput, commandLine.ShowFailureReport);
callback = new ParallelRunnerCallbackAdapter(callback);
var testOptions = new TestOptions
{
OpenInBrowser = commandLine.OpenInBrowser,
TestFileTimeoutMilliseconds = commandLine.TimeOutMilliseconds,
MaxDegreeOfParallelism = commandLine.Parallelism,
TestingMode = commandLine.TestMode,
CoverageOptions = new CoverageOptions
{
Enabled = commandLine.Coverage,
IncludePatterns = (commandLine.CoverageIncludePatterns ?? "").Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries),
ExcludePatterns = (commandLine.CoverageExcludePatterns ?? "").Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
}
};
if (!commandLine.Discovery)
{
testResultsSummary = testRunner.RunTests(commandLine.Files, testOptions, callback);
ProcessTestSummaryTransformers(commandLine, testResultsSummary);
}
else
{
Console.WriteLine("Test Discovery");
var tests = testRunner.DiscoverTests(commandLine.Files, testOptions).ToList();
Console.WriteLine("\nDiscovered {0} tests", tests.Count);
foreach (var test in tests)
{
Console.WriteLine("Test '{0}:{1}' from '{2}'", test.ModuleName, test.TestName, test.InputTestFile);
}
return(0);
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
var failedCount = testResultsSummary.FailedCount;
if (commandLine.FailOnError && testResultsSummary.Errors.Any())
{
return(failedCount > 0 ? failedCount : 1);
}
return(failedCount);
}
private ICoverageEngine SetupCodeCoverageEngine(
TestOptions options,
ChutzpahTestSettingsFile chutzpahTestSettings,
IFrameworkDefinition definition,
List<ReferencedFile> referencedFiles)
{
ICoverageEngine coverageEngine = GetConfiguredCoverageEngine(options, chutzpahTestSettings);
if (coverageEngine != null)
{
var deps = coverageEngine.GetFileDependencies(definition);
foreach (string item in deps)
{
string sourcePath = fileProbe.GetPathInfo(Path.Combine(Constants.TestFileFolder, item)).FullPath;
ChutzpahTracer.TraceInformation(
"Added code coverage dependency '{0}' to referenced files",
sourcePath);
referencedFiles.Add(
new ReferencedFile
{
IsLocal = true,
IsCodeCoverageDependency = true,
Path = sourcePath,
IncludeInTestHarness = true
});
}
}
return coverageEngine;
}
public TestContext BuildContext(PathInfo file, TestOptions options)
{
return(BuildContext(new List <PathInfo> {
file
}, options));
}
private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, ITestMethodRunnerCallback callback, bool debugEnabled)
{
var referencedFile = testContext.ReferencedFiles.SingleOrDefault(x => x.IsFileUnderTest);
var testIndex = 0;
var summary = new TestFileSummary(testContext.InputTestFile);
var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled) ||
(testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);
if (codeCoverageEnabled)
{
summary.CoverageObject = new CoverageData();
}
string line;
while ((line = stream.ReadLine()) != null)
{
if (debugEnabled)
{
Console.WriteLine(line);
}
var match = prefixRegex.Match(line);
if (!match.Success)
{
continue;
}
var type = match.Groups["type"].Value;
var json = match.Groups["json"].Value;
// Only update last event timestamp if it is an important event.
// Log and error could happen even though no test progress is made
if (!type.Equals("Log") && !type.Equals("Error"))
{
lastTestEvent = DateTime.Now;
}
try
{
JsTestCase jsTestCase = null;
switch (type)
{
case "FileStart":
callback.FileStarted(testContext.InputTestFile);
break;
case "CoverageObject":
var jsCov = jsonSerializer.Deserialize <JsCoverage>(json);
summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext);
break;
case "FileDone":
var jsFileDone = jsonSerializer.Deserialize <JsFileDone>(json);
summary.TimeTaken = jsFileDone.TimeTaken;
callback.FileFinished(testContext.InputTestFile, summary);
break;
case "TestStart":
jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
callback.TestStarted(jsTestCase.TestCase);
break;
case "TestDone":
jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
AddLineNumber(referencedFile, testIndex, jsTestCase);
testIndex++;
callback.TestFinished(jsTestCase.TestCase);
summary.AddTestCase(jsTestCase.TestCase);
break;
case "Log":
var log = jsonSerializer.Deserialize <JsLog>(json);
// This is an internal log message
if (log.Log.Message.StartsWith(internalLogPrefix))
{
ChutzpahTracer.TraceInformation("Phantom Log - {0}", log.Log.Message.Substring(internalLogPrefix.Length).Trim());
break;
}
log.Log.InputTestFile = testContext.InputTestFile;
callback.FileLog(log.Log);
summary.Logs.Add(log.Log);
break;
case "Error":
var error = jsonSerializer.Deserialize <JsError>(json);
error.Error.InputTestFile = testContext.InputTestFile;
callback.FileError(error.Error);
summary.Errors.Add(error.Error);
ChutzpahTracer.TraceError("Eror recieved from Phantom {0}", error.Error.Message);
break;
}
}
catch (SerializationException e)
{
// Ignore malformed json and move on
ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
}
}
return(summary);
}
public TestCaseStreamReadResult Read(TestCaseSource <string> testCaseSource, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback)
{
if (testCaseSource == null)
{
throw new ArgumentNullException(nameof(testCaseSource));
}
if (testOptions == null)
{
throw new ArgumentNullException(nameof(testOptions));
}
if (testContext == null)
{
throw new ArgumentNullException(nameof(testContext));
}
var testCaseStreamReadResult = new TestCaseStreamReadResult();
var codeCoverageEnabled = testOptions.CoverageOptions.ShouldRunCoverage(testContext.TestFileSettings.CodeCoverageExecutionMode);
var streamingTestFileContexts = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.Select(x => new StreamingTestFileContext(x, testContext, codeCoverageEnabled))
.ToList();
var deferredEvents = new List <Action <StreamingTestFileContext> >();
if (streamingTestFileContexts.Count == 1)
{
currentTestFileContext = streamingTestFileContexts.First();
}
testCaseSource.Subscribe((line) => ProcessLine(line, testContext, streamingTestFileContexts, deferredEvents, callback, testOptions.DebugEnabled));
var readerTask = testCaseSource.Open();
while ((readerTask.Status == TaskStatus.WaitingToRun ||
readerTask.Status == TaskStatus.WaitingForActivation ||
readerTask.Status == TaskStatus.Running) && testCaseSource.IsAlive)
{
Thread.Sleep(100);
}
if (readerTask.IsCompleted)
{
ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile);
testCaseStreamReadResult.TestFileSummaries = streamingTestFileContexts.Select(x => x.TestFileSummary).ToList();
}
else
{
// Since we timed out make sure we play the deferred events so we do not lose errors
// We will just attach these events to the first test context at this point since we do
// not know where they belong
PlayDeferredEvents(streamingTestFileContexts.FirstOrDefault(), deferredEvents);
// We timed out so kill the process and return an empty test file summary
ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - testCaseSource.LastTestEvent).TotalMilliseconds);
testCaseSource.Dispose();
testCaseStreamReadResult.TimedOut = true;
testCaseStreamReadResult.TestFileSummaries = testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList();
}
return(testCaseStreamReadResult);
}
public TestContext BuildContext(PathInfo file, TestOptions options)
{
if (file == null)
{
throw new ArgumentNullException("testFilePathInfo");
}
PathType testFileKind = file.Type;
string testFilePath = file.FullPath;
if (IsValidTestPathType(testFileKind))
{
throw new ArgumentException("Expecting a .js, .ts, .coffee or .html file or a url");
}
if (testFilePath == null)
{
throw new FileNotFoundException("Unable to find file: " + file.Path);
}
var testFileDirectory = Path.GetDirectoryName(testFilePath);
var chutzpahTestSettings = ChutzpahTestSettingsFile.Read(testFileDirectory, fileProbe, serializer);
string testFileText;
if (testFileKind == PathType.Url)
{
testFileText = httpClient.GetContent(testFilePath);
}
else
{
testFileText = fileSystem.GetText(testFilePath);
}
IFrameworkDefinition definition;
if (TryDetectFramework(testFileText, testFileKind, chutzpahTestSettings, out definition))
{
// For HTML test files we don't need to create a test harness to just return this file
if (testFileKind == PathType.Html || testFileKind == PathType.Url)
{
ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", testFileKind);
return new TestContext
{
InputTestFile = testFilePath,
TestHarnessPath = testFilePath,
IsRemoteHarness = testFileKind == PathType.Url,
TestRunner = definition.TestRunner,
};
}
var referencedFiles = new List<ReferencedFile>();
var temporaryFiles = new List<string>();
var fileUnderTest = GetFileUnderTest(testFilePath, chutzpahTestSettings);
referencedFiles.Add(fileUnderTest);
definition.Process(fileUnderTest);
referenceProcessor.GetReferencedFiles(referencedFiles, definition, testFileText, testFilePath, chutzpahTestSettings);
ProcessForFilesGeneration(referencedFiles, temporaryFiles, chutzpahTestSettings);
IEnumerable<string> deps = definition.FileDependencies;
var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles);
AddTestFrameworkDependencies(deps, referencedFiles);
string testHtmlFilePath = CreateTestHarness(
definition,
chutzpahTestSettings,
options,
testFilePath,
referencedFiles,
coverageEngine,
temporaryFiles);
return new TestContext
{
InputTestFile = testFilePath,
TestHarnessPath = testHtmlFilePath,
ReferencedJavaScriptFiles = referencedFiles,
TestRunner = definition.TestRunner,
TemporaryFiles = temporaryFiles,
TestFileSettings = chutzpahTestSettings
};
}
else
{
ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", testFilePath);
}
return null;
}
private TestRunConfiguration BuildTestRunConfiguration(IEnumerable<PathInfo> scriptPaths, TestOptions testOptions)
{
var testRunConfiguration = new TestRunConfiguration();
// Find all chutzpah.json files for the input files
// Then group files by their respective settings file
var testGroups = new List<List<PathInfo>>();
var fileSettingGroups = from path in scriptPaths
let settingsFile = testSettingsService.FindSettingsFile(path.FullPath, testOptions.ChutzpahSettingsFileEnvironments)
group path by settingsFile;
// Scan over the grouped test files and if this file is set up for batching we add those files
// as a group to be tested. Otherwise, we will explode them out individually so they get run in their
// own context
foreach (var group in fileSettingGroups)
{
if (group.Key.EnableTestFileBatching.Value)
{
testGroups.Add(group.ToList());
}
else
{
foreach (var path in group)
{
testGroups.Add(new List<PathInfo> { path });
}
}
}
testRunConfiguration.TestGroups = testGroups;
// Take the parallelism degree to be the minimum of any non-null setting in chutzpah.json
testRunConfiguration.MaxDegreeOfParallelism = fileSettingGroups.Min(x => x.Key.Parallelism);
// Enable tracing if any setting is true
testRunConfiguration.EnableTracing = fileSettingGroups.Any(x => x.Key.EnableTracing.HasValue && x.Key.EnableTracing.Value);
testRunConfiguration.TraceFilePath = fileSettingGroups.Select(x => x.Key.TraceFilePath).FirstOrDefault(x => !string.IsNullOrEmpty(x)) ?? testRunConfiguration.TraceFilePath;
return testRunConfiguration;
}
private List <List <PathInfo> > BuildTestingGroups(IEnumerable <PathInfo> scriptPaths, TestOptions testOptions)
{
// Find all chutzpah.json files for the input files
// Then group files by their respective settings file
var testGroups = new List <List <PathInfo> >();
var fileSettingGroups = from path in scriptPaths
let settingsFile = testSettingsService.FindSettingsFile(Path.GetDirectoryName(path.FullPath), testOptions.ChutzpahSettingsFileEnvironments)
group path by settingsFile;
// Scan over the grouped test files and if this file is set up for batching we add those files
// as a group to be tested. Otherwise, we will explode them out individually so they get run in their
// own context
foreach (var group in fileSettingGroups)
{
if (group.Key.EnableTestFileBatching.Value)
{
testGroups.Add(group.ToList());
}
else
{
foreach (var path in group)
{
testGroups.Add(new List <PathInfo> {
path
});
}
}
}
return(testGroups);
}
private void BuildTestContexts(
TestOptions options,
List<List<PathInfo>> scriptPathGroups,
ParallelOptions parallelOptions,
CancellationTokenSource cancellationSource,
int resultCount,
ConcurrentBag<TestContext> testContexts,
ITestMethodRunnerCallback callback,
TestCaseSummary overallSummary)
{
Parallel.ForEach(scriptPathGroups, parallelOptions, testFiles =>
{
var pathString = string.Join(",", testFiles.Select(x => x.FullPath));
ChutzpahTracer.TraceInformation("Building test context for {0}", pathString);
try
{
if (cancellationSource.IsCancellationRequested) return;
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFiles, options, out testContext))
{
testContexts.Add(testContext);
}
else
{
ChutzpahTracer.TraceWarning("Unable to build test context for {0}", pathString);
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
ChutzpahTracer.TraceError("File search limit hit!!!");
cancellationSource.Cancel();
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testFiles.Select(x => x.FullPath).FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during building test context for {0}", pathString);
}
finally
{
ChutzpahTracer.TraceInformation("Finished building test context for {0}", pathString);
}
});
}
public TestContext BuildContext(PathInfo file, TestOptions options)
{
if (file == null)
{
throw new ArgumentNullException("testFilePathInfo");
}
ChutzpahTracer.TraceInformation("Building test context for '{0}'", file.FullPath);
PathType testFileKind = file.Type;
string testFilePath = file.FullPath;
if (!IsValidTestPathType(testFileKind))
{
throw new ArgumentException("Expecting a .js, .ts, .coffee or .html file or a url");
}
if (testFilePath == null)
{
throw new FileNotFoundException("Unable to find file: " + file.Path);
}
var testFileDirectory = Path.GetDirectoryName(testFilePath);
var chutzpahTestSettings = settingsService.FindSettingsFile(testFileDirectory);
if (!IsTestPathIncluded(testFilePath, chutzpahTestSettings))
{
ChutzpahTracer.TraceInformation("Excluded test file '{0}' given chutzpah.json settings", testFilePath);
return(null);
}
string testFileText;
if (testFileKind == PathType.Url)
{
testFileText = httpClient.GetContent(testFilePath);
}
else
{
testFileText = fileSystem.GetText(testFilePath);
}
IFrameworkDefinition definition;
if (TryDetectFramework(testFileText, testFileKind, chutzpahTestSettings, out definition))
{
// For HTML test files we don't need to create a test harness to just return this file
if (testFileKind == PathType.Html || testFileKind == PathType.Url)
{
ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", testFileKind);
return(new TestContext
{
InputTestFile = testFilePath,
TestHarnessPath = testFilePath,
IsRemoteHarness = testFileKind == PathType.Url,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
});
}
var referencedFiles = new List <ReferencedFile>();
var temporaryFiles = new List <string>();
string inputTestFileDir = Path.GetDirectoryName(testFilePath);
var testHarnessDirectory = GetTestHarnessDirectory(chutzpahTestSettings, inputTestFileDir);
var fileUnderTest = GetFileUnderTest(testFilePath, chutzpahTestSettings);
referencedFiles.Add(fileUnderTest);
definition.Process(fileUnderTest, testFileText, chutzpahTestSettings);
referenceProcessor.GetReferencedFiles(referencedFiles, definition, testFileText, testFilePath, chutzpahTestSettings);
// This is the legacy way Chutzpah compiled files that are TypeScript or CoffeeScript
// Remaining but will eventually be deprecated and removed
ProcessForFilesGeneration(referencedFiles, temporaryFiles, chutzpahTestSettings);
IEnumerable <string> deps = definition.GetFileDependencies(chutzpahTestSettings);
var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles);
AddTestFrameworkDependencies(deps, referencedFiles);
return(new TestContext
{
FrameworkDefinition = definition,
CoverageEngine = coverageEngine,
InputTestFile = testFilePath,
TestHarnessDirectory = testHarnessDirectory,
ReferencedFiles = referencedFiles,
TestRunner = definition.GetTestRunner(chutzpahTestSettings),
TemporaryFiles = temporaryFiles,
TestFileSettings = chutzpahTestSettings
});
}
else
{
ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", testFilePath);
}
return(null);
}
private IList<TestFileSummary> InvokeTestRunner(string headlessBrowserPath,
TestOptions options,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext.TestHarnessPath, testContext.IsRemoteHarness);
string runnerArgs = BuildRunnerArgs(options, testContext, fileUrl, runnerPath, testExecutionMode);
Func<ProcessStream, IList<TestFileSummary>> streamProcessor =
processStream => testCaseStreamReaderFactory.Create().Read(processStream, options, testContext, callback, m_debugEnabled);
var processResult = process.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.Select(x => x.Errors).FirstOrDefault(), callback);
return processResult.Model;
}
private TestRunConfiguration BuildTestRunConfiguration(IEnumerable <PathInfo> scriptPaths, TestOptions testOptions)
{
var testRunConfiguration = new TestRunConfiguration();
// Find all chutzpah.json files for the input files
// Then group files by their respective settings file
var testGroups = new List <List <PathInfo> >();
var fileSettingGroups = from path in scriptPaths
let settingsFile = testSettingsService.FindSettingsFile(path.FullPath, testOptions.ChutzpahSettingsFileEnvironments)
group path by settingsFile;
// Scan over the grouped test files and if this file is set up for batching we add those files
// as a group to be tested. Otherwise, we will explode them out individually so they get run in their
// own context
foreach (var group in fileSettingGroups)
{
if (group.Key.EnableTestFileBatching.Value)
{
testGroups.Add(group.ToList());
}
else
{
foreach (var path in group)
{
testGroups.Add(new List <PathInfo> {
path
});
}
}
}
testRunConfiguration.TestGroups = testGroups;
// Take the parallelism degree to be the minimum of any non-null setting in chutzpah.json
testRunConfiguration.MaxDegreeOfParallelism = fileSettingGroups.Min(x => x.Key.Parallelism);
// Enable tracing if any setting is true
testRunConfiguration.EnableTracing = fileSettingGroups.Any(x => x.Key.EnableTracing.HasValue && x.Key.EnableTracing.Value);
testRunConfiguration.TraceFilePath = fileSettingGroups.Select(x => x.Key.TraceFilePath).FirstOrDefault(x => !string.IsNullOrEmpty(x)) ?? testRunConfiguration.TraceFilePath;
return(testRunConfiguration);
}
public TestContext GetTestContext(string testFile, TestOptions options)
{
if (string.IsNullOrEmpty(testFile)) return null;
return testContextBuilder.BuildContext(testFile, options);
}
private void BuildTestContexts(
TestOptions options,
List <List <PathInfo> > scriptPathGroups,
ParallelOptions parallelOptions,
CancellationTokenSource cancellationSource,
int resultCount,
ConcurrentBag <TestContext> testContexts,
ITestMethodRunnerCallback callback,
TestCaseSummary overallSummary)
{
Parallel.ForEach(scriptPathGroups, parallelOptions, testFiles =>
{
var pathString = string.Join(",", testFiles.Select(x => x.FullPath));
ChutzpahTracer.TraceInformation("Building test context for {0}", pathString);
try
{
if (cancellationSource.IsCancellationRequested)
{
return;
}
TestContext testContext;
resultCount++;
if (testContextBuilder.TryBuildContext(testFiles, options, out testContext))
{
testContexts.Add(testContext);
}
else
{
ChutzpahTracer.TraceWarning("Unable to build test context for {0}", pathString);
}
// Limit the number of files we can scan to attempt to build a context for
// This is important in the case of folder scanning where many JS files may not be
// test files.
if (resultCount >= options.FileSearchLimit)
{
ChutzpahTracer.TraceError("File search limit hit!!!");
cancellationSource.Cancel();
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testFiles.Select(x => x.FullPath).FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during building test context for {0}", pathString);
}
finally
{
ChutzpahTracer.TraceInformation("Finished building test context for {0}", pathString);
}
});
}
public IEnumerable <TestCase> DiscoverTests(IEnumerable <string> testPaths, TestOptions options, ITestMethodRunnerCallback callback)
{
var summary = ProcessTestPaths(testPaths, options, TestExecutionMode.Discovery, callback);
return(summary.Tests);
}
public TestCaseSummary RunTests(string testPath,
TestOptions options,
ITestMethodRunnerCallback callback = null)
{
return(RunTests(new[] { testPath }, options, callback));
}
private IList <TestFileSummary> ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, ITestMethodRunnerCallback callback, bool debugEnabled)
{
var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled) ||
(testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);
var testFileContexts = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.Select(x => new TestFileContext(x, testContext, codeCoverageEnabled))
.ToList();
var testIndex = 0;
string line;
TestFileContext currentTestFileContext = null;
if (testFileContexts.Count == 1)
{
currentTestFileContext = testFileContexts.First();
}
var deferredEvents = new List <Action <TestFileContext> >();
while ((line = stream.ReadLine()) != null)
{
if (debugEnabled)
{
Console.WriteLine(line);
}
var match = prefixRegex.Match(line);
if (!match.Success)
{
continue;
}
var type = match.Groups["type"].Value;
var json = match.Groups["json"].Value;
// Only update last event timestamp if it is an important event.
// Log and error could happen even though no test progress is made
if (!type.Equals("Log") && !type.Equals("Error"))
{
lastTestEvent = DateTime.Now;
}
try
{
switch (type)
{
case "FileStart":
FireFileStarted(callback, testContext);
break;
case "CoverageObject":
var jsCov = jsonSerializer.Deserialize <JsCoverage>(json);
if (currentTestFileContext == null)
{
deferredEvents.Add((fileContext) => FireCoverageObject(callback, fileContext, jsCov));
}
else
{
FireCoverageObject(callback, currentTestFileContext, jsCov);
}
break;
case "FileDone":
var jsFileDone = jsonSerializer.Deserialize <JsFileDone>(json);
FireFileFinished(callback, testContext.InputTestFilesString, testFileContexts, jsFileDone);
break;
case "TestStart":
var jsTestCaseStart = jsonSerializer.Deserialize <JsTestCase>(json);
TestFileContext newContext = null;
var testName = jsTestCaseStart.TestCase.TestName.Trim();
var moduleName = (jsTestCaseStart.TestCase.ModuleName ?? "").Trim();
var fileContexts = GetFileMatches(testName, testFileContexts);
if (fileContexts.Count == 0 && currentTestFileContext == null)
{
// If there are no matches and not file context has been used yet
// then just choose the first context
newContext = testFileContexts[0];
}
else if (fileContexts.Count == 0)
{
// If there is already a current context and no matches we just keep using that context
// unless this test name has been used already in the current context. In that case
// move to the next one that hasn't seen this file yet
var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName);
if (testAlreadySeenInCurrentContext)
{
newContext = testFileContexts.FirstOrDefault(x => !x.HasTestBeenSeen(moduleName, testName)) ?? currentTestFileContext;
}
}
else if (fileContexts.Count > 1)
{
// If we found the test has more than one file match
// try to choose the best match, otherwise just choose the first one
// If we have no file context yet take the first one
if (currentTestFileContext == null)
{
newContext = fileContexts.First();
}
else
{
// In this case we have an existing file context so we need to
// 1. Check to see if this test has been seen already on that context
// if so we need to try the next file context that matches it
// 2. If it is not seen yet in the current context and the current context
// is one of the matches then keep using it
var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName);
var currentContextInFileMatches = fileContexts.Any(x => x == currentTestFileContext);
if (!testAlreadySeenInCurrentContext && currentContextInFileMatches)
{
// Keep the current context
newContext = currentTestFileContext;
}
else
{
// Either take first not used context OR the first one
newContext = fileContexts.Where(x => !x.IsUsed).FirstOrDefault() ?? fileContexts.First();
}
}
}
else if (fileContexts.Count == 1)
{
// We found a unique match
newContext = fileContexts[0];
}
if (newContext != null && newContext != currentTestFileContext)
{
currentTestFileContext = newContext;
testIndex = 0;
}
currentTestFileContext.IsUsed = true;
currentTestFileContext.MarkTestSeen(moduleName, testName);
PlayDeferredEvents(currentTestFileContext, deferredEvents);
jsTestCaseStart.TestCase.InputTestFile = currentTestFileContext.ReferencedFile.Path;
callback.TestStarted(jsTestCaseStart.TestCase);
break;
case "TestDone":
var jsTestCaseDone = jsonSerializer.Deserialize <JsTestCase>(json);
var currentTestIndex = testIndex;
FireTestFinished(callback, currentTestFileContext, jsTestCaseDone, currentTestIndex);
testIndex++;
break;
case "Log":
var log = jsonSerializer.Deserialize <JsLog>(json);
if (currentTestFileContext != null)
{
FireLogOutput(callback, currentTestFileContext, log);
}
else
{
deferredEvents.Add((fileContext) => FireLogOutput(callback, fileContext, log));
}
break;
case "Error":
var error = jsonSerializer.Deserialize <JsError>(json);
if (currentTestFileContext != null)
{
FireErrorOutput(callback, currentTestFileContext, error);
}
else
{
deferredEvents.Add((fileContext) => FireErrorOutput(callback, fileContext, error));
}
break;
}
}
catch (SerializationException e)
{
// Ignore malformed json and move on
ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
}
}
return(testFileContexts.Select(x => x.TestFileSummary).ToList());
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
if (options.OpenInBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.InputTestFile);
process.LaunchFileInBrowser(testContext.TestHarnessPath);
}
else
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
var testSummary = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testContext.InputTestFile,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.InputTestFile);
testFileSummaries.Enqueue(testSummary);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFile,
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled && !options.OpenInBrowser)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile);
}
}
}
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token
};
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
public void SetupEnvironment(TestOptions testOptions, TestContext testContext)
{
}
private void ExecuteTestContexts(
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback,
ConcurrentBag <TestContext> testContexts,
ParallelOptions parallelOptions,
string headlessBrowserPath,
ConcurrentQueue <TestFileSummary> testFileSummaries,
TestCaseSummary overallSummary)
{
Parallel.ForEach(
testContexts,
parallelOptions,
testContext =>
{
ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
try
{
try
{
testHarnessBuilder.CreateTestHarness(testContext, options);
}
catch (IOException)
{
// Mark this creation failed so we do not try to clean it up later
// This is to work around a bug in TestExplorer that runs chutzpah in parallel on
// the same files
// TODO(mmanela): Re-evalute if this is needed once they fix that bug
testContext.TestHarnessCreationFailed = true;
ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
throw;
}
if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
{
ChutzpahTracer.TraceInformation(
"Launching test harness '{0}' for file '{1}' in a browser",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
// Allow override from command line.
var browserArgs = testContext.TestFileSettings.BrowserArguments;
if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
{
var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
browserArgs = new Dictionary <string, string>
{
{ Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
};
}
process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
}
else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
{
ChutzpahTracer.TraceInformation(
"Invoking headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
var testSummaries = InvokeTestRunner(
headlessBrowserPath,
options,
testContext,
testExecutionMode,
callback);
foreach (var testSummary in testSummaries)
{
ChutzpahTracer.TraceInformation(
"Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
testContext.TestHarnessPath,
testSummary.Path,
testSummary.PassedCount,
testSummary.FailedCount,
testSummary.Errors.Count);
ChutzpahTracer.TraceInformation(
"Finished running headless browser on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testSummary.Path);
testFileSummaries.Enqueue(testSummary);
}
}
else if (options.TestLaunchMode == TestLaunchMode.Custom)
{
if (options.CustomTestLauncher == null)
{
throw new ArgumentNullException("TestOptions.CustomTestLauncher");
}
ChutzpahTracer.TraceInformation(
"Launching custom test on test harness '{0}' for file '{1}'",
testContext.TestHarnessPath,
testContext.FirstInputTestFile);
options.CustomTestLauncher.LaunchTest(testContext);
}
else
{
Debug.Assert(false);
}
}
catch (Exception e)
{
var error = new TestError
{
InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
Message = e.ToString()
};
overallSummary.Errors.Add(error);
callback.FileError(error);
ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
}
finally
{
ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
}
});
// Clean up test context
foreach (var testContext in testContexts)
{
// Don't clean up context if in debug mode
if (!m_debugEnabled &&
!testContext.TestHarnessCreationFailed &&
options.TestLaunchMode != TestLaunchMode.FullBrowser &&
options.TestLaunchMode != TestLaunchMode.Custom)
{
try
{
ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
testContextBuilder.CleanupContext(testContext);
}
catch (Exception e)
{
ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
}
}
}
}
private List<List<PathInfo>> BuildTestingGroups(IEnumerable<PathInfo> scriptPaths, TestOptions testOptions)
{
// Find all chutzpah.json files for the input files
// Then group files by their respective settings file
var testGroups = new List<List<PathInfo>>();
var fileSettingGroups = from path in scriptPaths
let settingsFile = testSettingsService.FindSettingsFile(path.FullPath, testOptions.ChutzpahSettingsFileEnvironments)
group path by settingsFile;
// Scan over the grouped test files and if this file is set up for batching we add those files
// as a group to be tested. Otherwise, we will explode them out individually so they get run in their
// own context
foreach (var group in fileSettingGroups)
{
if (group.Key.EnableTestFileBatching.Value)
{
testGroups.Add(group.ToList());
}
else
{
foreach (var path in group)
{
testGroups.Add(new List<PathInfo> { path });
}
}
}
return testGroups;
}
public IEnumerable <TestCase> DiscoverTests(IEnumerable <string> testPaths, TestOptions options)
{
IList <TestError> testErrors;
return(DiscoverTests(testPaths, options, out testErrors));
}
static int RunTests(CommandLine commandLine, IEnumerable <SummaryTransformer> transformers)
{
var testRunner = TestRunner.Create(debugEnabled: commandLine.Debug);
if (commandLine.Trace)
{
ChutzpahTracer.AddFileListener();
}
Console.WriteLine();
TestCaseSummary testResultsSummary = null;
try
{
var callback = commandLine.TeamCity
? (ITestMethodRunnerCallback) new TeamCityConsoleRunnerCallback()
: new StandardConsoleRunnerCallback(commandLine.Silent, commandLine.VsOutput, commandLine.ShowFailureReport, commandLine.FailOnError);
callback = new ParallelRunnerCallbackAdapter(callback);
var testOptions = new TestOptions
{
TestLaunchMode = commandLine.OpenInBrowser ? TestLaunchMode.FullBrowser : TestLaunchMode.HeadlessBrowser,
Engine = commandLine.Engine,
OpenInBrowserName = commandLine.BrowserName,
OpenInBrowserArgs = commandLine.BrowserArgs,
TestFileTimeoutMilliseconds = commandLine.TimeOutMilliseconds,
ChutzpahSettingsFileEnvironments = commandLine.SettingsFileEnvironments,
Proxy = commandLine.Proxy,
CoverageOptions = new CoverageOptions
{
Enabled = commandLine.Coverage,
IncludePatterns = (commandLine.CoverageIncludePatterns ?? "").Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries),
ExcludePatterns = (commandLine.CoverageExcludePatterns ?? "").Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries),
IgnorePatterns = (commandLine.CoverageIgnorePatterns ?? "").Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
}
};
if (commandLine.Parallelism > 0)
{
testOptions.MaxDegreeOfParallelism = commandLine.Parallelism;
}
if (!commandLine.Discovery)
{
testResultsSummary = testRunner.RunTests(commandLine.Files, testOptions, callback);
ProcessTestSummaryTransformers(transformers, commandLine, testResultsSummary);
}
else
{
Console.WriteLine("Test Discovery");
var tests = testRunner.DiscoverTests(commandLine.Files, testOptions).ToList();
Console.WriteLine("\nDiscovered {0} tests", tests.Count);
foreach (var test in tests)
{
Console.WriteLine("Test '{0}:{1}' from '{2}'", test.ModuleName, test.TestName, test.InputTestFile);
}
return(0);
}
}
catch (ArgumentException ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex.Message);
Console.ResetColor();
}
if (testRunner.ActiveWebServerHost != null)
{
Console.WriteLine("Press any key to end Chutzpah...");
Console.ReadKey();
testRunner.ActiveWebServerHost.Dispose();
}
var failedCount = testResultsSummary.FailedCount;
if (commandLine.FailOnError && testResultsSummary.Errors.Any())
{
return(failedCount > 0 ? failedCount : 1);
}
return(failedCount);
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, options.MaxDegreeOfParallelism);
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token
};
var scriptPaths = FindTestFiles(testPaths, options);
// Build test contexts in parallel
BuildTestContexts(options, scriptPaths, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
compilerCache.Save();
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
public IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOptions options)
{
IList<TestError> testErrors;
return DiscoverTests(testPaths, options, out testErrors);
}
public bool TryBuildContext(IEnumerable <string> files, TestOptions options, out TestContext context)
{
context = BuildContext(files, options);
return(context != null);
}
public IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOptions options, out IList<TestError> errors)
{
var summary = ProcessTestPaths(testPaths, options, TestExecutionMode.Discovery, RunnerCallback.Empty);
errors = summary.Errors;
return summary.Tests;
}
public bool TryBuildContext(PathInfo file, TestOptions options, out TestContext context)
{
context = BuildContext(file, options);
return(context != null);
}
public TestCaseSummary RunTests(string testPath,
TestOptions options,
ITestMethodRunnerCallback callback = null)
{
return RunTests(new[] { testPath }, options, callback);
}
public void Run()
{
var phantomLocation = FindBrowser() ?? EnsurePhantom();
var testRunner = ChutzpahContainer.Current.GetInstance<TestRunner>();
if(setup.Debug)
testRunner.EnableDebugMode();
if (!string.IsNullOrEmpty(phantomLocation))
TestRunner.HeadlessBrowserName = phantomLocation;
var testOptions = new TestOptions
{
OpenInBrowser = setup.OpenInBrowser,
TestFileTimeoutMilliseconds = setup.Timeout,
MaxDegreeOfParallelism = setup.Parallelism,
TestingMode = GetTestingMode(setup.File)
};
var testResultsSummary = testRunner.RunTests(setup.File.FullName, testOptions, testRunnerCallback);
FailedCount = testResultsSummary.FailedCount;
Success = FailedCount == 0;
}
public TestCaseSummary RunTests(IEnumerable<string> testPaths,
TestOptions options,
ITestMethodRunnerCallback callback = null)
{
callback = options.TestLaunchMode == TestLaunchMode.FullBrowser || callback == null ? RunnerCallback.Empty : callback;
callback.TestSuiteStarted();
var summary = ProcessTestPaths(testPaths, options, TestExecutionMode.Execution, callback);
callback.TestSuiteFinished(summary);
return summary;
}
