private IList <ReferencedFile> GetReferencedFiles( HashSet <string> discoveredPaths, IFrameworkDefinition definition, string textToParse, string currentFilePath, ChutzpahTestSettingsFile chutzpahTestSettings) { var referencedFiles = new List <ReferencedFile>(); Regex regex = RegexPatterns.JsReferencePathRegex; foreach (Match match in regex.Matches(textToParse)) { if (!ShouldIncludeReference(match)) { continue; } string referencePath = match.Groups["Path"].Value; ProcessFilePathAsReference( discoveredPaths, definition, currentFilePath, chutzpahTestSettings, referencePath, referencedFiles, new ReferencePathSettings()); } foreach (Match match in RegexPatterns.JsTemplatePathRegex.Matches(textToParse)) { string referencePath = match.Groups["Path"].Value; referencePath = AdjustPathIfRooted(chutzpahTestSettings, referencePath); string relativeReferencePath = Path.Combine(Path.GetDirectoryName(currentFilePath), referencePath); string absoluteFilePath = fileProbe.FindFilePath(relativeReferencePath); if (referencedFiles.All(r => r.Path != absoluteFilePath)) { ChutzpahTracer.TraceInformation("Added html template '{0}' to referenced files", absoluteFilePath); referencedFiles.Add(new ReferencedFile { Path = absoluteFilePath, IsLocal = false, IncludeInTestHarness = true }); } } return(referencedFiles); }
private string ResolveFilePath(ChutzpahTestSettingsFile settings, string path) { string relativeLocationPath = Path.Combine(settings.SettingsFileDirectory, path ?? ""); string absoluteFilePath = fileProbe.FindFilePath(relativeLocationPath); return(absoluteFilePath); }
private string GetTestHarnessTemplatePath(IFrameworkDefinition definition, ChutzpahTestSettingsFile chutzpahTestSettings) { string templatePath = null; if (!string.IsNullOrEmpty(chutzpahTestSettings.CustomTestHarnessPath)) { // If CustomTestHarnessPath is absolute path then Path.Combine just returns it var harnessPath = Path.Combine(chutzpahTestSettings.SettingsFileDirectory, chutzpahTestSettings.CustomTestHarnessPath); var fullPath = fileProbe.FindFilePath(harnessPath); if (fullPath != null) { ChutzpahTracer.TraceInformation("Using Custom Test Harness from {0}", fullPath); templatePath = fullPath; } else { ChutzpahTracer.TraceError("Cannot find Custom Test Harness at {0}", chutzpahTestSettings.CustomTestHarnessPath); } } if (templatePath == null) { templatePath = fileProbe.GetPathInfo(Path.Combine(Constants.TestFileFolder, definition.GetTestHarness(chutzpahTestSettings))).FullPath; ChutzpahTracer.TraceInformation("Using builtin Test Harness from {0}", templatePath); } return(templatePath); }
public bool IsTestFile(string file, ChutzpahSettingsFileEnvironments environments = null) { ChutzpahTracer.TraceInformation("Determining if '{0}' might be a test file", file); if (string.IsNullOrWhiteSpace(file)) { return(false); } var testFilePath = fileProbe.FindFilePath(file); if (testFilePath == null) { ChutzpahTracer.TraceInformation("Rejecting '{0}' since either it doesnt exist", file); return(false); } var chutzpahTestSettings = settingsService.FindSettingsFile(testFilePath, environments); if (!IsTestPathIncluded(testFilePath, chutzpahTestSettings)) { ChutzpahTracer.TraceInformation("Excluded test file '{0}' given chutzpah.json settings", testFilePath); return(false); } // If the framework or tests filters are set in the settings file then no need to check for // test framework if (!string.IsNullOrEmpty(chutzpahTestSettings.Framework) || chutzpahTestSettings.Tests.Any()) { return(true); } else { string testFileText = fileSystem.GetText(testFilePath); IFrameworkDefinition definition; var frameworkDetetected = TryDetectFramework(testFileText, FileProbe.GetFilePathType(testFilePath), chutzpahTestSettings, out definition); if (frameworkDetetected) { ChutzpahTracer.TraceInformation("Assuming '{0}' is a test file", file); } return(frameworkDetetected); } }
public void SetupEnvironment(TestOptions testOptions, TestContext testContext) { if (testContext.TestFileSettings.Engine != Engine.Chrome || (testContext.TestFileSettings.EngineOptions != null && testContext.TestFileSettings.EngineOptions.PreventDownloadOfEngineDepenedencies)) { return; } string runnerPath = Directory.GetParent(fileProbe.FindFilePath(testContext.TestRunner)) + @"\setupRunner.js"; var environmentVariables = BuildEnvironmentVariables(); var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerPath, environmentVariables, out string standardOutput, out string standardError); if (!processResult) { throw new ChutzpahException($"Unable to install Chromium: Output: {standardOutput}, Error: {standardError}"); } }
public PhantomTestExecutionProvider(IProcessHelper process, IFileProbe fileProbe, IUrlBuilder urlBuilder, ITestCaseStreamReaderFactory readerFactory) { this.processTools = process; this.fileProbe = fileProbe; this.urlBuilder = urlBuilder; this.headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName); if (headlessBrowserPath == null) { throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName); } this.readerFactory = readerFactory; }
public NodeTestExecutionProvider(IProcessHelper process, IFileProbe fileProbe, IUrlBuilder urlBuilder, ITestCaseStreamReaderFactory readerFactory) { this.processTools = process; this.fileProbe = fileProbe; this.urlBuilder = urlBuilder; var path = Path.Combine("Node", Environment.Is64BitProcess ? "x64" : "x86", HeadlessBrowserName); this.headlessBrowserPath = fileProbe.FindFilePath(path); if (path == null) { throw new FileNotFoundException("Unable to find node: " + path); } this.readerFactory = readerFactory; isRunningElevated = process.IsRunningElevated(); }
public IList <TestFileSummary> Execute(TestOptions testOptions, TestContext testContext, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback) { string runnerPath = fileProbe.FindFilePath(testContext.TestRunner); string fileUrl = BuildHarnessUrl(testContext); string runnerArgs = BuildRunnerArgs(testOptions, testContext, fileUrl, runnerPath, testExecutionMode); var streamTimeout = ((testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500).GetValueOrDefault(); // Add buffer to timeout to account for serialization Func <ProcessStreamStringSource, TestCaseStreamReadResult> streamProcessor = processStream => readerFactory.Create().Read(processStream, testOptions, testContext, callback); var processResult = processTools.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor, streamTimeout, null); HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.TestFileSummaries.Select(x => x.Errors).FirstOrDefault(), callback); return(processResult.Model.TestFileSummaries); }
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths, TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback) { ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, options.MaxDegreeOfParallelism); options.TestExecutionMode = testExecutionMode; stopWatch.Start(); string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName); if (testPaths == null) { throw new ArgumentNullException("testPaths"); } if (headlessBrowserPath == null) { throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName); } if (fileProbe.FindFilePath(TestRunnerJsName) == null) { throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName); } var overallSummary = new TestCaseSummary(); // Concurrent list to collect test contexts var testContexts = new ConcurrentBag <TestContext>(); // Concurrent collection used to gather the parallel results from var testFileSummaries = new ConcurrentQueue <TestFileSummary>(); var resultCount = 0; var cancellationSource = new CancellationTokenSource(); var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = options.MaxDegreeOfParallelism, CancellationToken = cancellationSource.Token }; var scriptPaths = FindTestFiles(testPaths, options); // Build test contexts in parallel BuildTestContexts(options, scriptPaths, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary); // Compile the test contexts if (!PerformBatchCompile(callback, testContexts)) { return(overallSummary); } // Build test harness for each context and execute it in parallel ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary); // Gather TestFileSummaries into TaseCaseSummary foreach (var fileSummary in testFileSummaries) { overallSummary.Append(fileSummary); } stopWatch.Stop(); overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds); compilerCache.Save(); // Clear the settings file cache since in VS Chutzpah is not unloaded from memory. // If we don't clear then the user can never update the file. testSettingsService.ClearCache(); ChutzpahTracer.TraceInformation( "Chutzpah run finished with {0} passed, {1} failed and {2} errors", overallSummary.PassedCount, overallSummary.FailedCount, overallSummary.Errors.Count); return(overallSummary); }
private TestCaseSummary ProcessTestPaths(IEnumerable <string> testPaths, TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback) { options.TestExecutionMode = testExecutionMode; stopWatch.Start(); string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName); if (testPaths == null) { throw new ArgumentNullException("testPaths"); } if (headlessBrowserPath == null) { throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName); } if (fileProbe.FindFilePath(TestRunnerJsName) == null) { throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName); } var overallSummary = new TestCaseSummary(); // Concurrent list to collect test contexts var testContexts = new ConcurrentBag <TestContext>(); // Concurrent collection used to gather the parallel results from var testFileSummaries = new ConcurrentQueue <TestFileSummary>(); var resultCount = 0; var cancellationSource = new CancellationTokenSource(); // Given the input paths discover the potential test files var scriptPaths = FindTestFiles(testPaths, options); // Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled. // If so, we keep those tests in a group together to be used in one context // Otherwise, we put each file in its own test group so each get their own context var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options); ConfigureTracing(testRunConfiguration); var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue ? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value) : options.MaxDegreeOfParallelism; var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token }; ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism); // Build test contexts in parallel given a list of files each BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary); // Compile the test contexts if (!PerformBatchCompile(callback, testContexts)) { return(overallSummary); } // Build test harness for each context and execute it in parallel ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary); // Gather TestFileSummaries into TaseCaseSummary foreach (var fileSummary in testFileSummaries) { overallSummary.Append(fileSummary); } stopWatch.Stop(); overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds); overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary); // Clear the settings file cache since in VS Chutzpah is not unloaded from memory. // If we don't clear then the user can never update the file. testSettingsService.ClearCache(); ChutzpahTracer.TraceInformation( "Chutzpah run finished with {0} passed, {1} failed and {2} errors", overallSummary.PassedCount, overallSummary.FailedCount, overallSummary.Errors.Count); return(overallSummary); }
private IList <ReferencedFile> GetReferencedFiles( HashSet <string> discoveredPaths, IFrameworkDefinition definition, string textToParse, string currentFilePath, ChutzpahTestSettingsFile chutzpahTestSettings) { var referencedFiles = new List <ReferencedFile>(); Regex regex = RegexPatterns.JsReferencePathRegex; foreach (Match match in regex.Matches(textToParse)) { if (!ShouldIncludeReference(match)) { continue; } string referencePath = match.Groups["Path"].Value; ProcessFilePathAsReference( discoveredPaths, definition, currentFilePath, chutzpahTestSettings, referencePath, referencedFiles, new ReferencePathSettings()); } foreach (Match match in RegexPatterns.JsTemplatePathRegex.Matches(textToParse)) { string referencePath = null, templateId = null, templateType = null; TemplateMode templateMode = TemplateMode.Raw; for (var i = 0; i < match.Groups["PropName"].Captures.Count; i++) { var propName = match.Groups["PropName"].Captures[i].Value.ToLowerInvariant(); var propValue = match.Groups["PropValue"].Captures[i].Value; switch (propName) { case "path": referencePath = propValue; break; case "id": templateId = propValue; break; case "type": templateType = propValue; break; case "mode": if (propValue.Equals("script", StringComparison.OrdinalIgnoreCase)) { templateMode = TemplateMode.Script; } break; default: break; } } referencePath = AdjustPathIfRooted(chutzpahTestSettings, referencePath); string relativeReferencePath = Path.Combine(Path.GetDirectoryName(currentFilePath), referencePath); string absoluteFilePath = fileProbe.FindFilePath(relativeReferencePath); if (referencedFiles.All(r => r.Path != absoluteFilePath)) { ChutzpahTracer.TraceInformation("Added html template '{0}' to referenced files", absoluteFilePath); referencedFiles.Add(new ReferencedFile { Path = absoluteFilePath, IsLocal = false, IncludeInTestHarness = true, TemplateOptions = new TemplateOptions { Mode = templateMode, Id = templateId, Type = templateType } }); } } return(referencedFiles); }
/// <summary> /// Matches the current test path against the Tests settings. The first setting to accept a file wins. /// </summary> /// <param name="testFilePath"></param> /// <param name="chutzpahTestSettings"></param> /// <returns></returns> private bool IsTestPathIncluded(string testFilePath, ChutzpahTestSettingsFile chutzpahTestSettings) { // If those test filters are given then accept the test path if (!chutzpahTestSettings.Tests.Any()) { return(true); } testFilePath = FileProbe.NormalizeFilePath(testFilePath); foreach (var pathSettings in chutzpahTestSettings.Tests.Where(x => x != null)) { var includePattern = FileProbe.NormalizeFilePath(pathSettings.Include); var excludePattern = FileProbe.NormalizeFilePath(pathSettings.Exclude); // The path we assume default to the chuzpah.json directory if the Path property is not set var testPath = string.IsNullOrEmpty(pathSettings.Path) ? chutzpahTestSettings.SettingsFileDirectory : pathSettings.Path; testPath = FileProbe.NormalizeFilePath(testPath); testPath = testPath != null?Path.Combine(chutzpahTestSettings.SettingsFileDirectory, testPath) : null; // If a file path is given just match the test file against it to see if we should urn var filePath = fileProbe.FindFilePath(testPath); if (filePath != null) { if (filePath.Equals(testFilePath, StringComparison.OrdinalIgnoreCase)) { ChutzpahTracer.TraceInformation("Test file {0} matched test file path from settings file", testFilePath); return(true); } } // If a folder path is given then match the test file path that is in that folder with the optional include/exclude paths var folderPath = FileProbe.NormalizeFilePath(fileProbe.FindFolderPath(testPath)); if (folderPath != null) { if (testFilePath.Contains(folderPath)) { var shouldIncludeFile = (includePattern == null || NativeImports.PathMatchSpec(testFilePath, includePattern)) && (excludePattern == null || !NativeImports.PathMatchSpec(testFilePath, excludePattern)); if (shouldIncludeFile) { ChutzpahTracer.TraceInformation( "Test file {0} matched folder {1} with include {2} and exclude {3} patterns from settings file", testFilePath, folderPath, includePattern, excludePattern); return(true); } else { ChutzpahTracer.TraceInformation( "Test file {0} did not match folder {1} with include {2} and exclude {3} patterns from settings file", testFilePath, folderPath, includePattern, excludePattern); } } } } return(false); }