private IList <ReferencedFile> GetReferencedFiles(
HashSet <string> discoveredPaths,
IFrameworkDefinition definition,
string textToParse,
string currentFilePath,
ChutzpahTestSettingsFile chutzpahTestSettings)
{
var referencedFiles = new List <ReferencedFile>();
Regex regex = RegexPatterns.JsReferencePathRegex;
foreach (Match match in regex.Matches(textToParse))
{
if (!ShouldIncludeReference(match))
{
continue;
}
string referencePath = match.Groups["Path"].Value;
ProcessFilePathAsReference(
discoveredPaths,
definition,
currentFilePath,
chutzpahTestSettings,
referencePath,
referencedFiles,
new ReferencePathSettings());
}
foreach (Match match in RegexPatterns.JsTemplatePathRegex.Matches(textToParse))
{
string referencePath = match.Groups["Path"].Value;
referencePath = AdjustPathIfRooted(chutzpahTestSettings, referencePath);
string relativeReferencePath = Path.Combine(Path.GetDirectoryName(currentFilePath), referencePath);
string absoluteFilePath = fileProbe.FindFilePath(relativeReferencePath);
if (referencedFiles.All(r => r.Path != absoluteFilePath))
{
ChutzpahTracer.TraceInformation("Added html template '{0}' to referenced files", absoluteFilePath);
referencedFiles.Add(new ReferencedFile {
Path = absoluteFilePath, IsLocal = false, IncludeInTestHarness = true
});
}
}
return(referencedFiles);
}
private string ResolveFilePath(ChutzpahTestSettingsFile settings, string path)
{
string relativeLocationPath = Path.Combine(settings.SettingsFileDirectory, path ?? "");
string absoluteFilePath = fileProbe.FindFilePath(relativeLocationPath);
return(absoluteFilePath);
}
private string GetTestHarnessTemplatePath(IFrameworkDefinition definition, ChutzpahTestSettingsFile chutzpahTestSettings)
{
string templatePath = null;
if (!string.IsNullOrEmpty(chutzpahTestSettings.CustomTestHarnessPath))
{
// If CustomTestHarnessPath is absolute path then Path.Combine just returns it
var harnessPath = Path.Combine(chutzpahTestSettings.SettingsFileDirectory, chutzpahTestSettings.CustomTestHarnessPath);
var fullPath = fileProbe.FindFilePath(harnessPath);
if (fullPath != null)
{
ChutzpahTracer.TraceInformation("Using Custom Test Harness from {0}", fullPath);
templatePath = fullPath;
}
else
{
ChutzpahTracer.TraceError("Cannot find Custom Test Harness at {0}", chutzpahTestSettings.CustomTestHarnessPath);
}
}
if (templatePath == null)
{
templatePath = fileProbe.GetPathInfo(Path.Combine(Constants.TestFileFolder, definition.GetTestHarness(chutzpahTestSettings))).FullPath;
ChutzpahTracer.TraceInformation("Using builtin Test Harness from {0}", templatePath);
}
return(templatePath);
}
public bool IsTestFile(string file, ChutzpahSettingsFileEnvironments environments = null)
{
ChutzpahTracer.TraceInformation("Determining if '{0}' might be a test file", file);
if (string.IsNullOrWhiteSpace(file))
{
return(false);
}
var testFilePath = fileProbe.FindFilePath(file);
if (testFilePath == null)
{
ChutzpahTracer.TraceInformation("Rejecting '{0}' since either it doesnt exist", file);
return(false);
}
var chutzpahTestSettings = settingsService.FindSettingsFile(testFilePath, environments);
if (!IsTestPathIncluded(testFilePath, chutzpahTestSettings))
{
ChutzpahTracer.TraceInformation("Excluded test file '{0}' given chutzpah.json settings", testFilePath);
return(false);
}
// If the framework or tests filters are set in the settings file then no need to check for
// test framework
if (!string.IsNullOrEmpty(chutzpahTestSettings.Framework) || chutzpahTestSettings.Tests.Any())
{
return(true);
}
else
{
string testFileText = fileSystem.GetText(testFilePath);
IFrameworkDefinition definition;
var frameworkDetetected = TryDetectFramework(testFileText, FileProbe.GetFilePathType(testFilePath), chutzpahTestSettings, out definition);
if (frameworkDetetected)
{
ChutzpahTracer.TraceInformation("Assuming '{0}' is a test file", file);
}
return(frameworkDetetected);
}
}
public void SetupEnvironment(TestOptions testOptions, TestContext testContext)
{
if (testContext.TestFileSettings.Engine != Engine.Chrome ||
(testContext.TestFileSettings.EngineOptions != null && testContext.TestFileSettings.EngineOptions.PreventDownloadOfEngineDepenedencies))
{
return;
}
string runnerPath = Directory.GetParent(fileProbe.FindFilePath(testContext.TestRunner)) + @"\setupRunner.js";
var environmentVariables = BuildEnvironmentVariables();
var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerPath, environmentVariables, out string standardOutput, out string standardError);
if (!processResult)
{
throw new ChutzpahException($"Unable to install Chromium: Output: {standardOutput}, Error: {standardError}");
}
}
public PhantomTestExecutionProvider(IProcessHelper process, IFileProbe fileProbe,
IUrlBuilder urlBuilder, ITestCaseStreamReaderFactory readerFactory)
{
this.processTools = process;
this.fileProbe = fileProbe;
this.urlBuilder = urlBuilder;
this.headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
this.readerFactory = readerFactory;
}
public NodeTestExecutionProvider(IProcessHelper process, IFileProbe fileProbe,
IUrlBuilder urlBuilder, ITestCaseStreamReaderFactory readerFactory)
{
this.processTools = process;
this.fileProbe = fileProbe;
this.urlBuilder = urlBuilder;
var path = Path.Combine("Node", Environment.Is64BitProcess ? "x64" : "x86", HeadlessBrowserName);
this.headlessBrowserPath = fileProbe.FindFilePath(path);
if (path == null)
{
throw new FileNotFoundException("Unable to find node: " + path);
}
this.readerFactory = readerFactory;
isRunningElevated = process.IsRunningElevated();
}
public IList <TestFileSummary> Execute(TestOptions testOptions,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext);
string runnerArgs = BuildRunnerArgs(testOptions, testContext, fileUrl, runnerPath, testExecutionMode);
var streamTimeout = ((testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500).GetValueOrDefault(); // Add buffer to timeout to account for serialization
Func <ProcessStreamStringSource, TestCaseStreamReadResult> streamProcessor =
processStream => readerFactory.Create().Read(processStream, testOptions, testContext, callback);
var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor, streamTimeout, null);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.TestFileSummaries.Select(x => x.Errors).FirstOrDefault(), callback);
return(processResult.Model.TestFileSummaries);
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, options.MaxDegreeOfParallelism);
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token
};
var scriptPaths = FindTestFiles(testPaths, options);
// Build test contexts in parallel
BuildTestContexts(options, scriptPaths, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
compilerCache.Save();
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths,
TestOptions options,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
options.TestExecutionMode = testExecutionMode;
stopWatch.Start();
string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
if (testPaths == null)
{
throw new ArgumentNullException("testPaths");
}
if (headlessBrowserPath == null)
{
throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
}
if (fileProbe.FindFilePath(TestRunnerJsName) == null)
{
throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);
}
var overallSummary = new TestCaseSummary();
// Concurrent list to collect test contexts
var testContexts = new ConcurrentBag <TestContext>();
// Concurrent collection used to gather the parallel results from
var testFileSummaries = new ConcurrentQueue <TestFileSummary>();
var resultCount = 0;
var cancellationSource = new CancellationTokenSource();
// Given the input paths discover the potential test files
var scriptPaths = FindTestFiles(testPaths, options);
// Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
// If so, we keep those tests in a group together to be used in one context
// Otherwise, we put each file in its own test group so each get their own context
var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);
ConfigureTracing(testRunConfiguration);
var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
: options.MaxDegreeOfParallelism;
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token
};
ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);
// Build test contexts in parallel given a list of files each
BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);
// Compile the test contexts
if (!PerformBatchCompile(callback, testContexts))
{
return(overallSummary);
}
// Build test harness for each context and execute it in parallel
ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);
// Gather TestFileSummaries into TaseCaseSummary
foreach (var fileSummary in testFileSummaries)
{
overallSummary.Append(fileSummary);
}
stopWatch.Stop();
overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);
overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);
// Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
// If we don't clear then the user can never update the file.
testSettingsService.ClearCache();
ChutzpahTracer.TraceInformation(
"Chutzpah run finished with {0} passed, {1} failed and {2} errors",
overallSummary.PassedCount,
overallSummary.FailedCount,
overallSummary.Errors.Count);
return(overallSummary);
}
private IList <ReferencedFile> GetReferencedFiles(
HashSet <string> discoveredPaths,
IFrameworkDefinition definition,
string textToParse,
string currentFilePath,
ChutzpahTestSettingsFile chutzpahTestSettings)
{
var referencedFiles = new List <ReferencedFile>();
Regex regex = RegexPatterns.JsReferencePathRegex;
foreach (Match match in regex.Matches(textToParse))
{
if (!ShouldIncludeReference(match))
{
continue;
}
string referencePath = match.Groups["Path"].Value;
ProcessFilePathAsReference(
discoveredPaths,
definition,
currentFilePath,
chutzpahTestSettings,
referencePath,
referencedFiles,
new ReferencePathSettings());
}
foreach (Match match in RegexPatterns.JsTemplatePathRegex.Matches(textToParse))
{
string referencePath = null, templateId = null, templateType = null;
TemplateMode templateMode = TemplateMode.Raw;
for (var i = 0; i < match.Groups["PropName"].Captures.Count; i++)
{
var propName = match.Groups["PropName"].Captures[i].Value.ToLowerInvariant();
var propValue = match.Groups["PropValue"].Captures[i].Value;
switch (propName)
{
case "path":
referencePath = propValue;
break;
case "id":
templateId = propValue;
break;
case "type":
templateType = propValue;
break;
case "mode":
if (propValue.Equals("script", StringComparison.OrdinalIgnoreCase))
{
templateMode = TemplateMode.Script;
}
break;
default:
break;
}
}
referencePath = AdjustPathIfRooted(chutzpahTestSettings, referencePath);
string relativeReferencePath = Path.Combine(Path.GetDirectoryName(currentFilePath), referencePath);
string absoluteFilePath = fileProbe.FindFilePath(relativeReferencePath);
if (referencedFiles.All(r => r.Path != absoluteFilePath))
{
ChutzpahTracer.TraceInformation("Added html template '{0}' to referenced files", absoluteFilePath);
referencedFiles.Add(new ReferencedFile
{
Path = absoluteFilePath,
IsLocal = false,
IncludeInTestHarness = true,
TemplateOptions = new TemplateOptions
{
Mode = templateMode,
Id = templateId,
Type = templateType
}
});
}
}
return(referencedFiles);
}
/// <summary>
/// Matches the current test path against the Tests settings. The first setting to accept a file wins.
/// </summary>
/// <param name="testFilePath"></param>
/// <param name="chutzpahTestSettings"></param>
/// <returns></returns>
private bool IsTestPathIncluded(string testFilePath, ChutzpahTestSettingsFile chutzpahTestSettings)
{
// If those test filters are given then accept the test path
if (!chutzpahTestSettings.Tests.Any())
{
return(true);
}
testFilePath = FileProbe.NormalizeFilePath(testFilePath);
foreach (var pathSettings in chutzpahTestSettings.Tests.Where(x => x != null))
{
var includePattern = FileProbe.NormalizeFilePath(pathSettings.Include);
var excludePattern = FileProbe.NormalizeFilePath(pathSettings.Exclude);
// The path we assume default to the chuzpah.json directory if the Path property is not set
var testPath = string.IsNullOrEmpty(pathSettings.Path) ? chutzpahTestSettings.SettingsFileDirectory : pathSettings.Path;
testPath = FileProbe.NormalizeFilePath(testPath);
testPath = testPath != null?Path.Combine(chutzpahTestSettings.SettingsFileDirectory, testPath) : null;
// If a file path is given just match the test file against it to see if we should urn
var filePath = fileProbe.FindFilePath(testPath);
if (filePath != null)
{
if (filePath.Equals(testFilePath, StringComparison.OrdinalIgnoreCase))
{
ChutzpahTracer.TraceInformation("Test file {0} matched test file path from settings file", testFilePath);
return(true);
}
}
// If a folder path is given then match the test file path that is in that folder with the optional include/exclude paths
var folderPath = FileProbe.NormalizeFilePath(fileProbe.FindFolderPath(testPath));
if (folderPath != null)
{
if (testFilePath.Contains(folderPath))
{
var shouldIncludeFile = (includePattern == null || NativeImports.PathMatchSpec(testFilePath, includePattern)) &&
(excludePattern == null || !NativeImports.PathMatchSpec(testFilePath, excludePattern));
if (shouldIncludeFile)
{
ChutzpahTracer.TraceInformation(
"Test file {0} matched folder {1} with include {2} and exclude {3} patterns from settings file",
testFilePath,
folderPath,
includePattern,
excludePattern);
return(true);
}
else
{
ChutzpahTracer.TraceInformation(
"Test file {0} did not match folder {1} with include {2} and exclude {3} patterns from settings file",
testFilePath,
folderPath,
includePattern,
excludePattern);
}
}
}
}
return(false);
}
