private IEnumerable <PathInfo> FindTestFiles(IEnumerable <string> testPaths, TestOptions options) { IEnumerable <PathInfo> scriptPaths = Enumerable.Empty <PathInfo>(); // If the path list contains only chutzpah.json files then use those files for getting the list of test paths var testPathList = testPaths.ToList(); if (testPathList.All(testPath => Path.GetFileName(testPath).Equals(Constants.SettingsFileName, StringComparison.OrdinalIgnoreCase))) { ChutzpahTracer.TraceInformation("Using Chutzpah.json files to find tests"); foreach (var path in testPathList) { var chutzpahJsonPath = fileProbe.FindFilePath(path); if (chutzpahJsonPath == null) { ChutzpahTracer.TraceWarning("Supplied chutzpah.json path {0} does not exist", path); } // The FindSettingsFile api takes the directory of the file since it caches this for use in later test runs // this could be cleaned up to have two APIS one which lets you give the direct file var settingsFile = testSettingsService.FindSettingsFile(Path.GetDirectoryName(chutzpahJsonPath)); var pathInfos = fileProbe.FindScriptFiles(settingsFile); scriptPaths = scriptPaths.Concat(pathInfos); } } else { scriptPaths = fileProbe.FindScriptFiles(testPathList, options.TestingMode); } return(scriptPaths); }
private IEnumerable <PathInfo> FindTestFiles(IEnumerable <string> testPaths, TestOptions options) { IEnumerable <PathInfo> scriptPaths = Enumerable.Empty <PathInfo>(); // If the path list contains only chutzpah.json files then use those files for getting the list of test paths var testPathList = testPaths.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); if (testPathList.All(testPath => Path.GetFileName(testPath).Equals(Constants.SettingsFileName, StringComparison.OrdinalIgnoreCase))) { ChutzpahTracer.TraceInformation("Using Chutzpah.json files to find tests"); foreach (var path in testPathList) { var chutzpahJsonPath = fileProbe.FindFilePath(path); if (chutzpahJsonPath == null) { ChutzpahTracer.TraceWarning("Supplied chutzpah.json path {0} does not exist", path); } var settingsFile = testSettingsService.FindSettingsFile(chutzpahJsonPath, options.ChutzpahSettingsFileEnvironments); var pathInfos = fileProbe.FindScriptFiles(settingsFile); scriptPaths = scriptPaths.Concat(pathInfos); } } else { scriptPaths = fileProbe.FindScriptFiles(testPathList); } return(scriptPaths .Where(x => x.FullPath != null) .ToList());; }
private void BuildTestContexts( TestOptions options, IEnumerable <PathInfo> scriptPaths, ParallelOptions parallelOptions, CancellationTokenSource cancellationSource, int resultCount, ConcurrentBag <TestContext> testContexts, ITestMethodRunnerCallback callback, TestCaseSummary overallSummary) { Parallel.ForEach(scriptPaths, parallelOptions, testFile => { ChutzpahTracer.TraceInformation("Building test context for {0}", testFile.FullPath); try { if (cancellationSource.IsCancellationRequested) { return; } TestContext testContext; resultCount++; if (testContextBuilder.TryBuildContext(testFile, options, out testContext)) { testContexts.Add(testContext); } else { ChutzpahTracer.TraceWarning("Unable to build test context for {0}", testFile.FullPath); } // Limit the number of files we can scan to attempt to build a context for // This is important in the case of folder scanning where many JS files may not be // test files. if (resultCount >= options.FileSearchLimit) { ChutzpahTracer.TraceError("File search limit hit!!!"); cancellationSource.Cancel(); } } catch (Exception e) { var error = new TestError { InputTestFile = testFile.FullPath, Message = e.ToString() }; overallSummary.Errors.Add(error); callback.FileError(error); ChutzpahTracer.TraceError(e, "Error during building test context for {0}", testFile.FullPath); } finally { ChutzpahTracer.TraceInformation("Finished building test context for {0}", testFile.FullPath); } }); }
private void ResolveAMDBaseUrl(ChutzpahTestSettingsFile settings, IDictionary <string, string> chutzpahVariables) { if (!string.IsNullOrEmpty(settings.AMDBasePath)) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.AMDBasePath)); settings.AMDBasePath = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDBasePath)) { ChutzpahTracer.TraceWarning("Unable to find AMDBasePath at {0}", settings.AMDBasePath); } } if (!string.IsNullOrEmpty(settings.AMDBaseUrlOverride)) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.AMDBaseUrlOverride)); settings.AMDBaseUrlOverride = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDBaseUrlOverride)) { ChutzpahTracer.TraceWarning("Unable to find AMDBaseUrlOverride at {0}", settings.AMDBaseUrlOverride); } } }
private void ResolveAMDBaseUrl(ChutzpahTestSettingsFile settings) { if (!string.IsNullOrEmpty(settings.AMDBasePath)) { string absoluteFilePath = ResolveFolderPath(settings, settings.AMDBasePath); settings.AMDBasePath = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDBasePath)) { ChutzpahTracer.TraceWarning("Unable to find AMDBasePath at {0}", settings.AMDBasePath); } } }
public IEnumerable <PathInfo> FindScriptFiles(IEnumerable <string> testPaths, TestingMode testingMode) { if (testPaths == null) { yield break; } foreach (var path in testPaths) { var pathInfo = GetPathInfo(path); switch (pathInfo.Type) { case PathType.Url: if (testingMode == TestingMode.HTML || testingMode == TestingMode.All) { yield return(pathInfo); } break; case PathType.Html: case PathType.JavaScript: case PathType.CoffeeScript: case PathType.TypeScript: case PathType.TypeScriptDef: if (!testingMode.FileBelongsToTestingMode(path)) { break; } yield return(pathInfo); break; case PathType.Folder: var query = from file in fileSystem.GetFiles(pathInfo.FullPath, "*.*", SearchOption.AllDirectories) where file.Length < 260 && !IsTemporaryChutzpahFile(file) && testingMode.FileBelongsToTestingMode(file) select file; foreach (var item in query) { yield return(GetPathInfo(item)); } break; default: ChutzpahTracer.TraceWarning("Ignoring unsupported test path '{0}'", path); break; } } }
private void ResolveTestHarnessDirectory(ChutzpahTestSettingsFile settings, IDictionary <string, string> chutzpahVariables) { if (settings.TestHarnessLocationMode == TestHarnessLocationMode.Custom) { if (settings.TestHarnessDirectory != null) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.TestHarnessDirectory)); settings.TestHarnessDirectory = absoluteFilePath; } if (settings.TestHarnessDirectory == null) { settings.TestHarnessLocationMode = TestHarnessLocationMode.TestFileAdjacent; ChutzpahTracer.TraceWarning("Unable to find custom test harness directory at {0}", settings.TestHarnessDirectory); } } }
public IEnumerable <PathInfo> FindScriptFiles(IEnumerable <string> testPaths) { if (testPaths == null) { yield break; } foreach (var path in testPaths) { var pathInfo = GetPathInfo(path); switch (pathInfo.Type) { case PathType.Url: yield return(pathInfo); break; case PathType.Html: case PathType.JavaScript: yield return(pathInfo); break; case PathType.Folder: var query = from file in fileSystem.GetFiles(pathInfo.FullPath, "*.*", SearchOption.AllDirectories) where file.Length < 260 && !IsTemporaryChutzpahFile(file) let info = GetPathInfo(file) where info.Type != PathType.Other select info; foreach (var item in query) { yield return(item); } break; default: ChutzpahTracer.TraceWarning("Ignoring unsupported test path '{0}'", path); break; } } }
private void ResolveAMDBaseUrl(ChutzpahTestSettingsFile settings, IDictionary <string, string> chutzpahVariables) { if (!string.IsNullOrEmpty(settings.AMDBaseUrl)) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.AMDBaseUrl)); settings.AMDBaseUrl = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDBaseUrl)) { ChutzpahTracer.TraceWarning("Unable to find AMDBaseUrl at {0}", settings.AMDBaseUrl); } } if (!string.IsNullOrEmpty(settings.AMDAppDirectory)) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.AMDAppDirectory)); settings.AMDAppDirectory = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDAppDirectory)) { ChutzpahTracer.TraceWarning("Unable to find AMDAppDirectory at {0}", settings.AMDAppDirectory); } } // Legacy AMDBasePath property if (!string.IsNullOrEmpty(settings.AMDBasePath)) { string absoluteFilePath = ResolveFolderPath(settings, ExpandVariable(chutzpahVariables, settings.AMDBasePath)); settings.AMDBasePath = absoluteFilePath; if (string.IsNullOrEmpty(settings.AMDBasePath)) { ChutzpahTracer.TraceWarning("Unable to find AMDBasePath at {0}", settings.AMDBasePath); } } }
public TestContext BuildContext(IEnumerable <PathInfo> files, TestOptions options) { if (files == null) { throw new ArgumentNullException("testFilePathInfo"); } if (!files.Any()) { ChutzpahTracer.TraceInformation("No files given to build test context for"); return(null); } var fileCount = files.Count(); var allFilePathString = string.Join(",", files.Select(x => x.FullPath)); ChutzpahTracer.TraceInformation("Building test context for {0}", allFilePathString); // Make sure all test paths have been resolved to real files var missingPaths = files.Where(x => x.FullPath == null).ToList(); if (missingPaths.Any()) { throw new FileNotFoundException("Unable to find files: " + string.Join(",", missingPaths.Select(x => x.Path))); } // Make sure all test paths have a valid file type if (!files.Select(x => x.Type).All(IsValidTestPathType)) { throw new ArgumentException("Expecting valid file or a url"); } if (fileCount > 1 && files.Any(x => x.Type == PathType.Url || x.Type == PathType.Html)) { throw new InvalidOperationException("Cannot build a batch context for Url or Html test files"); } // We use the first file's directory to find the chutzpah.json file and to test framework // since we assume all files in the batch must have the same values for those PathType firstFileKind = files.First().Type; string firstFilePath = files.First().FullPath; var chutzpahTestSettings = settingsService.FindSettingsFile(firstFilePath, options.ChutzpahSettingsFileEnvironments); // Exclude any files that are not included based on the settings file SettingsFileTestPath matchingTestSettingsPath; var testedPaths = files.Select(f => new PathWithTestSetting { File = f, IsIncluded = IsTestPathIncluded(f.FullPath, chutzpahTestSettings, out matchingTestSettingsPath), MatchingTestSetting = matchingTestSettingsPath }).ToList(); if (testedPaths.Any(x => !x.IsIncluded)) { var pathString = string.Join(",", testedPaths.Where(x => !x.IsIncluded).Select(x => x.File.FullPath)); ChutzpahTracer.TraceInformation("Excluding test files {0} given chutzpah.json settings", pathString); testedPaths = testedPaths.Where(x => x.IsIncluded).ToList(); if (!testedPaths.Any()) { return(null); } } IFrameworkDefinition definition; if (TryDetectFramework(testedPaths.First().File, chutzpahTestSettings, out definition)) { // For HTML test files we don't need to create a test harness to just return this file if (firstFileKind == PathType.Html || firstFileKind == PathType.Url) { ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", firstFileKind); return(new TestContext { ReferencedFiles = new List <ReferencedFile> { new ReferencedFile { IsFileUnderTest = true, Path = firstFilePath } }, InputTestFiles = new[] { firstFilePath }, FirstInputTestFile = firstFilePath, InputTestFilesString = firstFilePath, TestHarnessPath = firstFilePath, IsRemoteHarness = firstFileKind == PathType.Url, TestRunner = definition.GetTestRunner(chutzpahTestSettings), }); } var temporaryFiles = new List <string>(); string firstInputTestFileDir = Path.GetDirectoryName(firstFilePath); var testHarnessDirectory = GetTestHarnessDirectory(chutzpahTestSettings, firstInputTestFileDir); var referencedFiles = GetFilesUnderTest(testedPaths, chutzpahTestSettings).ToList(); referenceProcessor.GetReferencedFiles(referencedFiles, definition, chutzpahTestSettings); IEnumerable <string> deps = definition.GetFileDependencies(chutzpahTestSettings); var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles); AddTestFrameworkDependencies(deps, referencedFiles); var testFiles = referencedFiles.Where(x => x.IsFileUnderTest).Select(x => x.Path).ToList(); return(new TestContext { FrameworkDefinition = definition, CoverageEngine = coverageEngine, InputTestFiles = testFiles, FirstInputTestFile = testFiles.FirstOrDefault(), InputTestFilesString = string.Join(",", testFiles), TestHarnessDirectory = testHarnessDirectory, ReferencedFiles = referencedFiles, TestRunner = definition.GetTestRunner(chutzpahTestSettings), TemporaryFiles = temporaryFiles, TestFileSettings = chutzpahTestSettings }); } else { ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", firstFilePath); } return(null); }
private void ExecuteTestContexts( TestOptions options, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback, ConcurrentBag <TestContext> testContexts, ParallelOptions parallelOptions, string headlessBrowserPath, ConcurrentQueue <TestFileSummary> testFileSummaries, TestCaseSummary overallSummary) { Parallel.ForEach( testContexts, parallelOptions, testContext => { ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode); try { try { testHarnessBuilder.CreateTestHarness(testContext, options); } catch (IOException) { // Mark this creation failed so we do not try to clean it up later // This is to work around a bug in TestExplorer that runs chutzpah in parallel on // the same files // TODO(mmanela): Re-evalute if this is needed once they fix that bug testContext.TestHarnessCreationFailed = true; ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile); throw; } if (options.TestLaunchMode == TestLaunchMode.FullBrowser) { ChutzpahTracer.TraceInformation( "Launching test harness '{0}' for file '{1}' in a browser", testContext.TestHarnessPath, testContext.FirstInputTestFile); // Allow override from command line. var browserArgs = testContext.TestFileSettings.BrowserArguments; if (!string.IsNullOrWhiteSpace(options.BrowserArgs)) { var path = BrowserPathHelper.GetBrowserPath(options.BrowserName); browserArgs = new Dictionary <string, string> { { Path.GetFileNameWithoutExtension(path), options.BrowserArgs } }; } process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs); } else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser) { ChutzpahTracer.TraceInformation( "Invoking headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.FirstInputTestFile); var testSummaries = InvokeTestRunner( headlessBrowserPath, options, testContext, testExecutionMode, callback); foreach (var testSummary in testSummaries) { ChutzpahTracer.TraceInformation( "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors", testContext.TestHarnessPath, testSummary.Path, testSummary.PassedCount, testSummary.FailedCount, testSummary.Errors.Count); ChutzpahTracer.TraceInformation( "Finished running headless browser on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testSummary.Path); testFileSummaries.Enqueue(testSummary); } } else if (options.TestLaunchMode == TestLaunchMode.Custom) { if (options.CustomTestLauncher == null) { throw new ArgumentNullException("TestOptions.CustomTestLauncher"); } ChutzpahTracer.TraceInformation( "Launching custom test on test harness '{0}' for file '{1}'", testContext.TestHarnessPath, testContext.FirstInputTestFile); options.CustomTestLauncher.LaunchTest(testContext); } else { Debug.Assert(false); } } catch (Exception e) { var error = new TestError { InputTestFile = testContext.InputTestFiles.FirstOrDefault(), Message = e.ToString() }; overallSummary.Errors.Add(error); callback.FileError(error); ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile); } finally { ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode); } }); // Clean up test context foreach (var testContext in testContexts) { // Don't clean up context if in debug mode if (!m_debugEnabled && !testContext.TestHarnessCreationFailed && options.TestLaunchMode != TestLaunchMode.FullBrowser && options.TestLaunchMode != TestLaunchMode.Custom) { try { ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile); testContextBuilder.CleanupContext(testContext); } catch (Exception e) { ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile); } } } }
private void ProcessFilePathAsReference( HashSet <string> discoveredPaths, IFrameworkDefinition definition, string relativeProcessingPath, ChutzpahTestSettingsFile chutzpahTestSettings, string referencePath, List <ReferencedFile> referencedFiles, ReferencePathSettings pathSettings) { ChutzpahTracer.TraceInformation("Investigating reference file path '{0}'", referencePath); // Check test settings and adjust the path if it is rooted (e.g. /some/path) referencePath = AdjustPathIfRooted(chutzpahTestSettings, referencePath); var referenceUri = new Uri(referencePath, UriKind.RelativeOrAbsolute); string referenceFileName = Path.GetFileName(referencePath); // Ignore test runner, since we use our own. if (definition.ReferenceIsDependency(referenceFileName, chutzpahTestSettings)) { ChutzpahTracer.TraceInformation( "Ignoring reference file '{0}' as a duplicate reference to {1}", referenceFileName, definition.FrameworkKey); return; } var isRelativeUri = !referenceUri.IsAbsoluteUri; // If this either a relative uri or a file uri if (isRelativeUri || referenceUri.IsFile) { var relativeProcessingPathFolder = fileSystem.FolderExists(relativeProcessingPath) ? relativeProcessingPath : Path.GetDirectoryName(relativeProcessingPath); string relativeReferencePath = Path.Combine(relativeProcessingPathFolder, referencePath); // Check if reference is a file string absoluteFilePath = fileProbe.FindFilePath(relativeReferencePath); if (absoluteFilePath != null) { VisitReferencedFile(absoluteFilePath, definition, discoveredPaths, referencedFiles, chutzpahTestSettings, pathSettings); return; } // Check if reference is a folder string absoluteFolderPath = fileProbe.FindFolderPath(relativeReferencePath); if (absoluteFolderPath != null) { var includePatterns = pathSettings.Includes.Select(x => UrlBuilder.NormalizeFilePath(x)).ToList(); var excludePatterns = pathSettings.Excludes.Select(x => UrlBuilder.NormalizeFilePath(x)).ToList(); // Find all files in this folder including sub-folders. This can be ALOT of files. // Only a subset of these files Chutzpah might understand so many of these will be ignored. var childFiles = fileSystem.GetFiles(absoluteFolderPath, "*.*", SearchOption.AllDirectories); var validFiles = from file in childFiles let normalizedFile = UrlBuilder.NormalizeFilePath(file) where !fileProbe.IsTemporaryChutzpahFile(file) && (!includePatterns.Any() || includePatterns.Any(pat => NativeImports.PathMatchSpec(normalizedFile, pat))) && (!excludePatterns.Any() || !excludePatterns.Any(pat => NativeImports.PathMatchSpec(normalizedFile, pat))) select file; validFiles.ForEach(file => VisitReferencedFile(file, definition, discoveredPaths, referencedFiles, chutzpahTestSettings, pathSettings)); return; } // At this point we know that this file/folder does not exist! ChutzpahTracer.TraceWarning("Referenced file '{0}' which was resolved to '{1}' does not exist", referencePath, relativeReferencePath); } else if (referenceUri.IsAbsoluteUri) { var referencedFile = new ReferencedFile { Path = referencePath, IsLocal = false, IncludeInTestHarness = true, IsTestFrameworkFile = pathSettings.IsTestFrameworkFile, TemplateOptions = pathSettings.TemplateOptions }; ChutzpahTracer.TraceInformation( "Added file '{0}' to referenced files. Local: {1}, IncludeInTestHarness: {2}", referencedFile.Path, referencedFile.IsLocal, referencedFile.IncludeInTestHarness); referencedFiles.Add(referencedFile); } }
public TestContext BuildContext(PathInfo file, TestOptions options) { if (file == null) { throw new ArgumentNullException("testFilePathInfo"); } ChutzpahTracer.TraceInformation("Building test context for '{0}'", file.FullPath); PathType testFileKind = file.Type; string testFilePath = file.FullPath; if (!IsValidTestPathType(testFileKind)) { throw new ArgumentException("Expecting a .js, .ts, .coffee or .html file or a url"); } if (testFilePath == null) { throw new FileNotFoundException("Unable to find file: " + file.Path); } var testFileDirectory = Path.GetDirectoryName(testFilePath); var chutzpahTestSettings = settingsService.FindSettingsFile(testFileDirectory); if (!IsTestPathIncluded(testFilePath, chutzpahTestSettings)) { ChutzpahTracer.TraceInformation("Excluded test file '{0}' given chutzpah.json settings", testFilePath); return(null); } string testFileText; if (testFileKind == PathType.Url) { testFileText = httpClient.GetContent(testFilePath); } else { testFileText = fileSystem.GetText(testFilePath); } IFrameworkDefinition definition; if (TryDetectFramework(testFileText, testFileKind, chutzpahTestSettings, out definition)) { // For HTML test files we don't need to create a test harness to just return this file if (testFileKind == PathType.Html || testFileKind == PathType.Url) { ChutzpahTracer.TraceInformation("Test kind is {0} so we are trusting the supplied test harness and not building our own", testFileKind); return(new TestContext { InputTestFile = testFilePath, TestHarnessPath = testFilePath, IsRemoteHarness = testFileKind == PathType.Url, TestRunner = definition.GetTestRunner(chutzpahTestSettings), }); } var referencedFiles = new List <ReferencedFile>(); var temporaryFiles = new List <string>(); string inputTestFileDir = Path.GetDirectoryName(testFilePath); var testHarnessDirectory = GetTestHarnessDirectory(chutzpahTestSettings, inputTestFileDir); var fileUnderTest = GetFileUnderTest(testFilePath, chutzpahTestSettings); referencedFiles.Add(fileUnderTest); definition.Process(fileUnderTest, testFileText, chutzpahTestSettings); referenceProcessor.GetReferencedFiles(referencedFiles, definition, testFileText, testFilePath, chutzpahTestSettings); // This is the legacy way Chutzpah compiled files that are TypeScript or CoffeeScript // Remaining but will eventually be deprecated and removed ProcessForFilesGeneration(referencedFiles, temporaryFiles, chutzpahTestSettings); IEnumerable <string> deps = definition.GetFileDependencies(chutzpahTestSettings); var coverageEngine = SetupCodeCoverageEngine(options, chutzpahTestSettings, definition, referencedFiles); AddTestFrameworkDependencies(deps, referencedFiles); return(new TestContext { FrameworkDefinition = definition, CoverageEngine = coverageEngine, InputTestFile = testFilePath, TestHarnessDirectory = testHarnessDirectory, ReferencedFiles = referencedFiles, TestRunner = definition.GetTestRunner(chutzpahTestSettings), TemporaryFiles = temporaryFiles, TestFileSettings = chutzpahTestSettings }); } else { ChutzpahTracer.TraceWarning("Failed to detect test framework for '{0}'", testFilePath); } return(null); }