internal void InstallPackages() { DirectoryPath packagesPath = GetAbsolutePackagesPath(); Trace.Information($"Installing packages to {packagesPath.FullPath} (using {(UseLocalPackagesFolder ? "local" : "global")} packages folder)"); try { // Add the global default sources if requested if (UseGlobalPackageSources) { _sourceRepositories.AddGlobalDefaults(); } // Get the local repository SourceRepository localRepository = _sourceRepositories.CreateRepository(packagesPath.FullPath); // Get the package manager and repositories WyamFolderNuGetProject nuGetProject = new WyamFolderNuGetProject(_fileSystem, _assemblyLoader, _currentFramework, packagesPath.FullPath); NuGetPackageManager packageManager = new NuGetPackageManager(_sourceRepositories, _settings, packagesPath.FullPath) { PackagesFolderNuGetProject = nuGetProject }; IReadOnlyList <SourceRepository> remoteRepositories = _sourceRepositories.GetDefaultRepositories(); // Resolve all the versions IReadOnlyList <SourceRepository> installationRepositories = remoteRepositories; try { ResolveVersions(localRepository, remoteRepositories); } catch (Exception ex) { Trace.Verbose($"Exception while resolving package versions: {ex.Message}"); Trace.Warning("Error while resolving package versions, attempting without remote repositories"); installationRepositories = new[] { localRepository }; ResolveVersions(localRepository, Array.Empty <SourceRepository>()); } // Install the packages (doing this synchronously since doing it in parallel triggers file lock errors in NuGet on a clean system) try { InstallPackages(packageManager, installationRepositories); } catch (Exception ex) { Trace.Verbose($"Exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}"); Trace.Warning("Error while installing packages, attempting without remote repositories"); InstallPackages(packageManager, new[] { localRepository }); } // Process the package (do this after all packages have been installed) nuGetProject.ProcessAssembliesAndContent(); } catch (Exception ex) { Trace.Verbose($"Unexpected exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}"); Trace.Warning("Error while installing packages, attempting to continue anyway"); } }
// Internal for testing internal void AddRecipePackageAndSetTheme() { if (Recipe == null && !string.IsNullOrEmpty(RecipeName)) { KnownRecipe knownRecipe; if (KnownRecipe.Values.TryGetValue(RecipeName, out knownRecipe)) { Trace.Verbose($"Recipe {RecipeName} was in the lookup of known recipes"); // Make sure we're not ignoring packages if (!IgnoreKnownRecipePackages) { // Add the package, but only if it wasn't added manually if (!string.IsNullOrEmpty(knownRecipe.PackageId) && !PackageInstaller.ContainsPackage(knownRecipe.PackageId)) { PackageInstaller.AddPackage(knownRecipe.PackageId, versionRange: $"[{Engine.Version}]", allowPrereleaseVersions: true); } } else { Trace.Verbose("Ignoring known recipe packages"); } // Set the theme if we don't already have one if (string.IsNullOrEmpty(Theme)) { Theme = knownRecipe.DefaultTheme; } } else { Trace.Verbose($"Recipe {RecipeName} is not in the lookup of known recipes"); } } }
/// <summary> /// Adds a package. /// </summary> /// <param name="packageId">The package identifier.</param> /// <param name="packageSources">The package sources.</param> /// <param name="versionRange">The version range.</param> /// <param name="getLatest">If set to <c>true</c>, the latest version of the package will always be downloaded.</param> /// <param name="allowPrereleaseVersions">If set to <c>true</c>, allow prerelease versions.</param> /// <param name="allowUnlisted">If set to <c>true</c>, allow unlisted versions.</param> /// <param name="exclusive">If set to <c>true</c>, only use the package sources defined for this package.</param> public void AddPackage( string packageId, IEnumerable <string> packageSources = null, string versionRange = null, bool getLatest = false, bool allowPrereleaseVersions = false, bool allowUnlisted = false, bool exclusive = false) { if (_packages.ContainsKey(packageId)) { throw new ArgumentException($"A package with the ID {packageId} has already been added"); } // If this package is a known Wyam extension and no version is specified, set to the current version if (KnownExtension.Values.Values.Any(x => x.PackageId == packageId) && versionRange == null) { versionRange = $"[{Engine.Version}]"; Trace.Verbose($"Added known extension package {packageId} without version, setting version to {versionRange}"); } _packages.Add( packageId, new Package( _currentFramework, packageId, packageSources?.Select(_sourceRepositories.CreateRepository).ToList(), versionRange, getLatest, allowPrereleaseVersions, allowUnlisted, exclusive)); }
public ExitCode Run(Preprocessor preprocessor) { if (GlobalArguments) { // Set verbose tracing if (_verbose) { Trace.Level = System.Diagnostics.SourceLevels.Verbose; } // Attach if (_attach) { Trace.Information($"Waiting for a debugger to attach to process {Process.GetCurrentProcess().Id} (or press a key to continue)..."); while (!Debugger.IsAttached && !Console.KeyAvailable) { Thread.Sleep(100); } if (Console.KeyAvailable) { Console.ReadKey(true); Trace.Information("Key pressed, continuing execution"); } else { Trace.Information("Debugger attached, continuing execution"); } } } return(RunCommand(preprocessor)); }
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter) { if (IsEnabled(logLevel)) { Trace.TraceEvent(TraceMapping[logLevel], $"{_categoryName}: {formatter(state, exception)}"); } }
public void BaseSetUp() { Directory.SetCurrentDirectory(TestContext.CurrentContext.TestDirectory); TestTraceListener listener = new TestTraceListener(TestContext.CurrentContext.Test.ID); _listeners.AddOrUpdate(TestContext.CurrentContext.Test.ID, listener, (x, y) => listener); Trace.AddListener(Listener); }
public void RemoveListener() { TestTraceListener listener = Listener; if (listener != null) { Trace.RemoveListener(listener); } }
public void RemoveListener() { TestTraceListener listener = Trace.Listeners.OfType <TestTraceListener>() .FirstOrDefault(x => x.TestId == TestContext.CurrentContext.Test.ID); if (listener != null) { Trace.RemoveListener(listener); } }
private void CatalogClasses() { Stopwatch stopwatch = Stopwatch.StartNew(); using (Trace.WithIndent().Information("Cataloging classes")) { ClassCatalog.CatalogTypes(AssemblyLoader.DirectAssemblies); stopwatch.Stop(); Trace.Information($"Classes cataloged in {stopwatch.ElapsedMilliseconds} ms"); } }
private void InstallPackages(FilePath configFilePath) { Stopwatch stopwatch = Stopwatch.StartNew(); using (Trace.WithIndent().Information("Installing NuGet packages")) { PackageInstaller.InstallPackages(configFilePath); stopwatch.Stop(); Trace.Information($"NuGet packages installed in {stopwatch.ElapsedMilliseconds} ms"); } }
private void LoadAssemblies() { Stopwatch stopwatch = Stopwatch.StartNew(); using (Trace.WithIndent().Information("Loading assemblies")) { AssemblyLoader.Load(); stopwatch.Stop(); Trace.Information($"Assemblies loaded in {stopwatch.ElapsedMilliseconds} ms"); } }
private static void UnhandledExceptionEvent(object sender, UnhandledExceptionEventArgs e) { // Exit with a error exit code Exception exception = e.ExceptionObject as Exception; if (exception != null) { Trace.Critical(exception.Message); Trace.Verbose(exception.ToString()); } Environment.Exit((int)ExitCode.UnhandledError); }
/// <inheritdoc /> public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { // Expire the internal Razor cache if this is a new execution // This needs to be done so that layouts/partials can be re-rendered if they've changed, // otherwise Razor will just use the previously cached version of them if (_executionId != Guid.Empty && _executionId != context.ExecutionId) { RazorService.ExpireChangeTokens(); } _executionId = context.ExecutionId; // Eliminate input documents that we shouldn't process List <IDocument> validInputs = inputs .Where(context, x => _ignorePrefix == null || !x.ContainsKey(Keys.SourceFileName) || !x.FilePath(Keys.SourceFileName).FullPath.StartsWith(_ignorePrefix)) .ToList(); if (validInputs.Count < inputs.Count) { Trace.Information($"Ignoring {inputs.Count - validInputs.Count} inputs due to source file name prefix"); } // Compile and evaluate the pages in parallel return(validInputs.AsParallel().Select(context, input => { Trace.Verbose("Processing Razor for {0}", input.SourceString()); Stream contentStream = context.GetContentStream(); using (Stream inputStream = input.GetStream()) { FilePath viewStartLocationPath = _viewStartPath?.Invoke <FilePath>(input, context); RenderRequest request = new RenderRequest { Input = inputStream, Output = contentStream, BaseType = _basePageType, Context = context, Document = input, LayoutLocation = _layoutPath?.Invoke <FilePath>(input, context)?.FullPath, ViewStartLocation = viewStartLocationPath != null ? GetRelativePath(viewStartLocationPath, context) : null, RelativePath = GetRelativePath(input, context), Model = _model == null ? input : _model.Invoke(input, context), }; RazorService.Render(request); } return context.GetDocument(input, contentStream); })); }
// Internal for testing internal void AddThemePackagesAndPath() { string inputPath = Theme; if (!string.IsNullOrEmpty(Theme)) { KnownTheme knownTheme; if (KnownTheme.Values.TryGetValue(Theme, out knownTheme)) { Trace.Verbose($"Theme {Theme} was in the lookup of known themes"); inputPath = knownTheme.InputPath; // Do a sanity check against the recipe (but only if we didn't explicitly specify one) if (Recipe == null && !string.IsNullOrEmpty(RecipeName) && !string.IsNullOrEmpty(knownTheme.Recipe) && !string.Equals(RecipeName, knownTheme.Recipe, StringComparison.OrdinalIgnoreCase)) { Trace.Warning($"Theme {Theme} is designed for recipe {knownTheme.Recipe} but is being used with recipe {RecipeName}, results may be unexpected"); } // Make sure we're not ignoring theme packages if (!IgnoreKnownThemePackages) { // Add any packages needed for the theme if (knownTheme.PackageIds != null) { foreach (string themePackageId in knownTheme.PackageIds.Where(x => !PackageInstaller.ContainsPackage(x))) { PackageInstaller.AddPackage(themePackageId, allowPrereleaseVersions: true); } } } else { Trace.Verbose("Ignoring known theme packages"); } } else { Trace.Verbose($"Theme {Theme} is not in the lookup of known themes, assuming it's an input path"); } } // Insert the theme path if (!string.IsNullOrEmpty(inputPath)) { _engine.FileSystem.InputPaths.Insert(0, new DirectoryPath(inputPath)); } }
private int Run(string[] args) { // Add a default trace listener Trace.AddListener(new SimpleColorConsoleTraceListener { TraceOutputOptions = System.Diagnostics.TraceOptions.None }); // Output version info Trace.Information($"Wyam version {Engine.Version}"); // It's not a serious console app unless there's some ASCII art OutputLogo(); // Make sure we're not running under Mono if (Type.GetType("Mono.Runtime") != null) { Trace.Critical("The Mono runtime is not supported. Please check the GitHub repository and issue tracker for information on .NET Core support for cross platform execution."); return((int)ExitCode.UnsupportedRuntime); } // Parse the command line Preprocessor preprocessor = new Preprocessor(); Command command; try { bool hasParseArgsErrors; command = CommandParser.Parse(args, preprocessor, out hasParseArgsErrors); if (command == null) { return(hasParseArgsErrors ? (int)ExitCode.CommandLineError : (int)ExitCode.Normal); } } catch (Exception ex) { Trace.Error("Error while parsing command line: {0}", ex.Message); if (Trace.Level == SourceLevels.Verbose) { Trace.Error("Stack trace:{0}{1}", Environment.NewLine, ex.StackTrace); } return((int)ExitCode.CommandLineError); } // Run the command return((int)command.Run(preprocessor)); }
private void Evaluate(string code) { if (string.IsNullOrEmpty(code)) { return; } Stopwatch stopwatch = Stopwatch.StartNew(); using (Trace.WithIndent().Information("Evaluating configuration script")) { CacheManager cacheManager = new CacheManager(_engine, _scriptManager, ConfigDllPath, ConfigHashPath, OutputScriptPath); cacheManager.EvaluateCode(code, ClassCatalog.GetClasses <IModule>().ToList(), OutputScript, IgnoreConfigHash, NoOutputConfigAssembly); stopwatch.Stop(); Trace.Information($"Evaluated configuration script in {stopwatch.ElapsedMilliseconds} ms"); } }
private int Run(string[] args) { // Add a default trace listener Trace.AddListener(new SimpleColorConsoleTraceListener { TraceOutputOptions = System.Diagnostics.TraceOptions.None }); // Output version info AssemblyInformationalVersionAttribute versionAttribute = Attribute.GetCustomAttribute(typeof(Program).Assembly, typeof(AssemblyInformationalVersionAttribute)) as AssemblyInformationalVersionAttribute; Trace.Information("Wyam version {0}", versionAttribute == null ? "unknown" : versionAttribute.InformationalVersion); // It's not a serious console app unless there's some ASCII art OutputLogo(); // Parse the command line Preprocessor preprocessor = new Preprocessor(); Command command; try { bool hasParseArgsErrors; command = CommandParser.Parse(args, preprocessor, out hasParseArgsErrors); if (command == null) { return(hasParseArgsErrors ? (int)ExitCode.CommandLineError : (int)ExitCode.Normal); } } catch (Exception ex) { Trace.Error("Error while parsing command line: {0}", ex.Message); if (Trace.Level == SourceLevels.Verbose) { Trace.Error("Stack trace:{0}{1}", Environment.NewLine, ex.StackTrace); } return((int)ExitCode.CommandLineError); } // Run the command return((int)command.Run(preprocessor)); }
/// <inheritdoc /> public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { #pragma warning disable RCS1163 // Unused parameter. // Handle invalid HTTPS certificates and allow alternate security protocols (see http://stackoverflow.com/a/5670954/807064) ServicePointManager.ServerCertificateValidationCallback = (s, cert, chain, ssl) => true; #pragma warning restore RCS1163 // Unused parameter. // Key = link, Value = source, tag HTML ConcurrentDictionary <string, ConcurrentBag <(string documentSource, string outerHtml)> > links = new ConcurrentDictionary <string, ConcurrentBag <(string documentSource, string outerHtml)> >(); // Key = source, Value = tag HTML ConcurrentDictionary <string, ConcurrentBag <string> > failures = new ConcurrentDictionary <string, ConcurrentBag <string> >(); // Gather all links HtmlParser parser = new HtmlParser(); context.ParallelForEach(inputs, input => GatherLinks(input, parser, links)); // This policy will limit the number of executing link validations. // Limit the amount of concurrent link checks to avoid overwhelming servers. Task[] tasks = links.Select( async link => { // Attempt to parse the URI if (!Uri.TryCreate(link.Key, UriKind.RelativeOrAbsolute, out Uri uri)) { AddOrUpdateFailure(link.Value, failures); } // Adjustment for double-slash link prefix which means use http:// or https:// depending on current protocol // The Uri class treats these as relative, but they're really absolute if (uri.ToString().StartsWith("//") && !Uri.TryCreate($"http:{link.Key}", UriKind.Absolute, out uri)) { AddOrUpdateFailure(link.Value, failures); } // Relative if (!uri.IsAbsoluteUri && _validateRelativeLinks && !(await ValidateRelativeLink(uri, context).ConfigureAwait(false))) { AddOrUpdateFailure(link.Value, failures); } // Absolute if (uri.IsAbsoluteUri && _validateAbsoluteLinks && !(await ValidateAbsoluteLink(uri, context).ConfigureAwait(false))) { AddOrUpdateFailure(link.Value, failures); } }).ToArray(); Task.WaitAll(tasks); // Report failures if (failures.Count > 0) { int failureCount = failures.Sum(x => x.Value.Count); string failureMessage = string.Join( Environment.NewLine, failures.Select(x => $"{x.Key}{Environment.NewLine} - {string.Join(Environment.NewLine + " - ", x.Value)}")); Trace.TraceEvent( _asError ? TraceEventType.Error : TraceEventType.Warning, $"{failureCount} link validation failures:{Environment.NewLine}{failureMessage}"); } return(inputs); }
protected override ExitCode RunCommand(Preprocessor preprocessor) { // Get the standard input stream _configOptions.Stdin = StandardInputReader.Read(); // Fix the root folder and other files DirectoryPath currentDirectory = Environment.CurrentDirectory; _configOptions.RootPath = _configOptions.RootPath == null ? currentDirectory : currentDirectory.Combine(_configOptions.RootPath); _logFilePath = _logFilePath == null ? null : _configOptions.RootPath.CombineFile(_logFilePath); _configOptions.ConfigFilePath = _configOptions.RootPath.CombineFile(_configOptions.ConfigFilePath ?? "config.wyam"); // Set up the log file if (_logFilePath != null) { Trace.AddListener(new SimpleFileTraceListener(_logFilePath.FullPath)); } // Get the engine and configurator EngineManager engineManager = EngineManager.Get(preprocessor, _configOptions); if (engineManager == null) { return(ExitCode.CommandLineError); } // Configure and execute if (!engineManager.Configure()) { return(ExitCode.ConfigurationError); } if (_verifyConfig) { Trace.Information("No errors. Exiting."); return(ExitCode.Normal); } Trace.Information($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.RootPath}"); Trace.Information($"Input path(s):{Environment.NewLine} {string.Join(Environment.NewLine + " ", engineManager.Engine.FileSystem.InputPaths)}"); Trace.Information($"Output path:{Environment.NewLine} {engineManager.Engine.FileSystem.OutputPath}"); if (!engineManager.Execute()) { return(ExitCode.ExecutionError); } bool messagePump = false; // Start the preview server IDisposable previewServer = null; if (_preview) { messagePump = true; DirectoryPath previewPath = _previewRoot == null ? engineManager.Engine.FileSystem.GetOutputDirectory().Path : engineManager.Engine.FileSystem.GetOutputDirectory(_previewRoot).Path; previewServer = PreviewServer.Start(previewPath, _previewPort, _previewForceExtension); } // Start the watchers IDisposable inputFolderWatcher = null; IDisposable configFileWatcher = null; if (_watch) { messagePump = true; Trace.Information("Watching paths(s) {0}", string.Join(", ", engineManager.Engine.FileSystem.InputPaths)); inputFolderWatcher = new ActionFileSystemWatcher(engineManager.Engine.FileSystem.GetOutputDirectory().Path, engineManager.Engine.FileSystem.GetInputDirectories().Select(x => x.Path), true, "*.*", path => { _changedFiles.Enqueue(path); _messageEvent.Set(); }); if (_configOptions.ConfigFilePath != null) { Trace.Information("Watching configuration file {0}", _configOptions.ConfigFilePath); configFileWatcher = new ActionFileSystemWatcher(engineManager.Engine.FileSystem.GetOutputDirectory().Path, new[] { _configOptions.ConfigFilePath.Directory }, false, _configOptions.ConfigFilePath.FileName.FullPath, path => { FilePath filePath = new FilePath(path); if (_configOptions.ConfigFilePath.Equals(filePath)) { _newEngine.Set(); _messageEvent.Set(); } }); } } // Start the message pump if an async process is running ExitCode exitCode = ExitCode.Normal; if (messagePump) { // Only wait for a key if console input has not been redirected, otherwise it's on the caller to exit if (!Console.IsInputRedirected) { // Start the key listening thread var thread = new Thread(() => { Trace.Information("Hit any key to exit"); Console.ReadKey(); _exit.Set(); _messageEvent.Set(); }) { IsBackground = true }; thread.Start(); } // Wait for activity while (true) { _messageEvent.WaitOne(); // Blocks the current thread until a signal if (_exit) { break; } // See if we need a new engine if (_newEngine) { // Get a new engine Trace.Information("Configuration file {0} has changed, re-running", _configOptions.ConfigFilePath); engineManager.Dispose(); engineManager = EngineManager.Get(preprocessor, _configOptions); // Configure and execute if (!engineManager.Configure()) { exitCode = ExitCode.ConfigurationError; break; } Console.WriteLine($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.RootPath}"); Console.WriteLine($"Input path(s):{Environment.NewLine} {string.Join(Environment.NewLine + " ", engineManager.Engine.FileSystem.InputPaths)}"); Console.WriteLine($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.OutputPath}"); if (!engineManager.Execute()) { exitCode = ExitCode.ExecutionError; break; } // Clear the changed files since we just re-ran string changedFile; while (_changedFiles.TryDequeue(out changedFile)) { } _newEngine.Unset(); } else { // Execute if files have changed HashSet <string> changedFiles = new HashSet <string>(); string changedFile; while (_changedFiles.TryDequeue(out changedFile)) { if (changedFiles.Add(changedFile)) { Trace.Verbose("{0} has changed", changedFile); } } if (changedFiles.Count > 0) { Trace.Information("{0} files have changed, re-executing", changedFiles.Count); if (!engineManager.Execute()) { exitCode = ExitCode.ExecutionError; break; } } } // Check one more time for exit if (_exit) { break; } Trace.Information("Hit any key to exit"); _messageEvent.Reset(); } // Shutdown Trace.Information("Shutting down"); engineManager.Dispose(); inputFolderWatcher?.Dispose(); configFileWatcher?.Dispose(); previewServer?.Dispose(); } return(exitCode); }
protected override void ParseOptions(ArgumentSyntax syntax) { syntax.DefineOption("w|watch", ref _watch, "Watches the input folder for any changes."); _preview = syntax.DefineOption("p|preview", ref _previewPort, false, "Start the preview web server on the specified port (default is " + _previewPort + ").").IsSpecified; if (syntax.DefineOption("force-ext", ref _previewForceExtension, "Force the use of extensions in the preview web server (by default, extensionless URLs may be used).").IsSpecified&& !_preview) { syntax.ReportError("force-ext can only be specified if the preview server is running."); } if (syntax.DefineOption("virtual-dir", ref _previewVirtualDirectory, DirectoryPath.FromString, "Serve files in the preview web server under the specified virtual directory.").IsSpecified&& !_preview) { syntax.ReportError("virtual-dir can only be specified if the preview server is running."); } if (syntax.DefineOption("preview-root", ref _previewRoot, DirectoryPath.FromString, "The path to the root of the preview server, if not the output folder.").IsSpecified&& !_preview) { syntax.ReportError("preview-root can only be specified if the preview server is running."); } syntax.DefineOptionList("i|input", ref _configOptions.InputPaths, DirectoryPath.FromString, "The path(s) of input files, can be absolute or relative to the current folder."); syntax.DefineOption("o|output", ref _configOptions.OutputPath, DirectoryPath.FromString, "The path to output files, can be absolute or relative to the current folder."); syntax.DefineOption("c|config", ref _configOptions.ConfigFilePath, FilePath.FromString, "Configuration file (by default, config.wyam is used)."); syntax.DefineOption("u|update-packages", ref _configOptions.UpdatePackages, "Check the NuGet server for more recent versions of each package and update them if applicable."); syntax.DefineOption("use-local-packages", ref _configOptions.UseLocalPackages, "Toggles the use of a local NuGet packages folder."); syntax.DefineOption("use-global-sources", ref _configOptions.UseGlobalSources, "Toggles the use of the global NuGet sources (default is false)."); syntax.DefineOption("packages-path", ref _configOptions.PackagesPath, DirectoryPath.FromString, "The packages path to use (only if use-local is true)."); syntax.DefineOption("output-script", ref _configOptions.OutputScript, "Outputs the config script after it's been processed for further debugging."); syntax.DefineOption("verify-config", ref _verifyConfig, false, "Compile the configuration but do not execute."); syntax.DefineOption("noclean", ref _configOptions.NoClean, "Prevents cleaning of the output path on each execution."); syntax.DefineOption("nocache", ref _configOptions.NoCache, "Prevents caching information during execution (less memory usage but slower execution)."); _logFilePath = $"wyam-{DateTime.Now:yyyyMMddHHmmssfff}.txt"; if (!syntax.DefineOption("l|log", ref _logFilePath, FilePath.FromString, false, "Log all trace messages to the specified log file (by default, wyam-[datetime].txt).").IsSpecified) { _logFilePath = null; } // Metadata // TODO: Remove this dictionary and initial/global options Dictionary <string, object> settingsDictionary = new Dictionary <string, object>(); IReadOnlyList <string> globalMetadata = null; if (syntax.DefineOptionList("g|global", ref globalMetadata, "Deprecated, do not use.").IsSpecified) { Trace.Warning("-g/--global is deprecated and will be removed in a future version. Please use -s/--setting instead."); AddSettings(settingsDictionary, globalMetadata); } IReadOnlyList <string> initialMetadata = null; if (syntax.DefineOptionList("initial", ref initialMetadata, "Deprecated, do not use.").IsSpecified) { Trace.Warning("--initial is deprecated and will be removed in a future version. Please use -s/--setting instead."); AddSettings(settingsDictionary, initialMetadata); } IReadOnlyList <string> settings = null; if (syntax.DefineOptionList("s|setting", ref settings, "Specifies a setting as a key=value pair. Use the syntax [x,y] to specify an array value.").IsSpecified) { // _configOptions.Settings = MetadataParser.Parse(settings); TODO: Use this when AddSettings() is removed AddSettings(settingsDictionary, settings); } if (settingsDictionary.Count > 0) { _configOptions.Settings = settingsDictionary; } }
/// <inheritdoc /> public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { HtmlParser parser = new HtmlParser(); using (IJsEnginePool enginePool = context.GetJsEnginePool(x => { if (string.IsNullOrWhiteSpace(_highlightJsFile)) { x.ExecuteResource("highlight-all.js", typeof(Highlight)); } else { x.ExecuteFile(_highlightJsFile); } })) { return(inputs.AsParallel().Select(context, input => { // We materialize the list before exiting the using statement, so safe to access enginePool // ReSharper disable once AccessToDisposedClosure using (IJsEngine engine = enginePool.GetEngine()) { try { using (Stream stream = input.GetStream()) using (IHtmlDocument htmlDocument = parser.Parse(stream)) { foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector)) { // Don't highlight anything that potentially is already highlighted if (element.ClassList.Contains("hljs")) { continue; } // Make sure to use TextContent, otherwise you'll get escaped html which highlight.js won't parse engine.SetVariableValue("input", element.TextContent); // Check if they specified a language in their code block string language = element.ClassList.FirstOrDefault(i => i.StartsWith("language")); try { if (language != null) { engine.SetVariableValue("language", language.Replace("language-", "")); engine.Execute("result = hljs.highlight(language, input)"); } else { language = "(auto)"; // set this to auto in case there is an exception below engine.Execute("result = hljs.highlightAuto(input)"); string detectedLanguage = engine.Evaluate <string>("result.language"); if (string.IsNullOrWhiteSpace(detectedLanguage) == false) { element.ClassList.Add("language-" + detectedLanguage); } } element.ClassList.Add("hljs"); string formatted = engine.Evaluate <string>("result.value"); element.InnerHtml = formatted; } catch (Exception innerEx) { if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage) { Trace.Warning("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message); } else { Trace.Information("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message); } } } string content = htmlDocument.ToHtml(); return context.GetDocument(input, content); } } catch (Exception ex) { Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message); return input; } } }).ToList()); } }
/// <inheritdoc /> public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { HtmlParser parser = new HtmlParser(); using (IJavaScriptEnginePool enginePool = context.GetJavaScriptEnginePool(x => { if (string.IsNullOrWhiteSpace(_highlightJsFile)) { x.ExecuteResource("highlight-all.js", typeof(Highlight)); } else { x.ExecuteFile(_highlightJsFile); } })) { return(inputs.AsParallel().Select(context, input => { try { using (Stream stream = input.GetStream()) { using (IHtmlDocument htmlDocument = parser.Parse(stream)) { foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector)) { // Don't highlight anything that potentially is already highlighted if (element.ClassList.Contains("hljs")) { continue; } try { HighlightElement(enginePool, element); } catch (Exception innerEx) { if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage) { Trace.Warning($"Exception while highlighting source code: {innerEx.Message}"); } else { Trace.Information($"Exception while highlighting source code: {innerEx.Message}"); } } } Stream contentStream = context.GetContentStream(); using (StreamWriter writer = contentStream.GetWriter()) { htmlDocument.ToHtml(writer, HtmlMarkupFormatter.Instance); writer.Flush(); return context.GetDocument(input, contentStream); } } } } catch (Exception ex) { Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message); return input; } }).ToList()); } }
// Internal for testing internal static bool ValidateRelativeLink(Uri uri, IExecutionContext context) { List <FilePath> checkPaths = new List <FilePath>(); // Remove the query string and fragment, if any string normalizedPath = uri.ToString(); if (normalizedPath.Contains("#")) { normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("#", StringComparison.Ordinal)); } if (normalizedPath.Contains("?")) { normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("?", StringComparison.Ordinal)); } if (normalizedPath == string.Empty) { return(true); } // Remove the link root if there is one and remove the preceding slash if (context.Settings.DirectoryPath(Keys.LinkRoot) != null && normalizedPath.StartsWith(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath)) { normalizedPath = normalizedPath.Substring(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath.Length); } if (normalizedPath.StartsWith("/")) { normalizedPath = normalizedPath.Length > 1 ? normalizedPath.Substring(1) : string.Empty; } // Add the base path if (normalizedPath != string.Empty) { checkPaths.Add(new FilePath(normalizedPath)); } // Add filenames checkPaths.AddRange(LinkGenerator.DefaultHidePages.Select(x => new FilePath(normalizedPath == string.Empty ? x : $"{normalizedPath}/{x}"))); // Add extensions checkPaths.AddRange(LinkGenerator.DefaultHideExtensions.SelectMany(x => checkPaths.Select(y => y.AppendExtension(x))).ToArray()); // Check all the candidate paths FilePath validatedPath = checkPaths.FirstOrDefault(x => { IFile outputFile; try { outputFile = context.FileSystem.GetOutputFile(x); } catch (Exception ex) { Trace.Warning($"Could not validate path {x.FullPath} for relative link {uri}: {ex.Message}"); return(false); } return(outputFile.Exists); }); if (validatedPath != null) { Trace.Verbose($"Validated relative link {uri} at {validatedPath.FullPath}"); return(true); } Trace.Warning($"Validation failure for relative link {uri}: could not find output file at any of {string.Join(", ", checkPaths.Select(x => x.FullPath))}"); return(false); }
// Internal for testing internal static bool ValidateAbsoluteLink(Uri uri, IExecutionContext context) { // Create a request HttpWebRequest request; try { request = WebRequest.Create(uri) as HttpWebRequest; } catch (NotSupportedException ex) { Trace.Warning($"Skipping absolute link {uri}: {ex.Message}"); return(true); } if (request == null) { Trace.Warning($"Skipping absolute link {uri}: only HTTP/HTTPS links are validated"); return(true); } // Set request properties request.Timeout = 60000; // 60 seconds // Perform request as HEAD HttpWebResponse response; request.Method = "HEAD"; try { response = (HttpWebResponse)request.GetResponse(); response.Close(); } catch (WebException) { response = null; } // Check the status code if (response != null) { if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400) { Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}"); return(true); } } // Try one more time as GET request.Method = "GET"; try { response = (HttpWebResponse)request.GetResponse(); response.Close(); } catch (WebException ex) { Trace.Warning($"Validation failure for absolute link {uri}: {ex.Message}"); return(false); } // Check the status code if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400) { Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}"); return(true); } Trace.Warning($"Validation failure for absolute link {uri}: returned status code {(int)response.StatusCode} {response.StatusCode}"); return(false); }
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { // Handle invalid HTTPS certificates and allow alternate security protocols (see http://stackoverflow.com/a/5670954/807064) ServicePointManager.ServerCertificateValidationCallback = (s, cert, chain, ssl) => true; // Key = link, Value = source, tag HTML ConcurrentDictionary <string, ConcurrentBag <Tuple <FilePath, string> > > links = new ConcurrentDictionary <string, ConcurrentBag <Tuple <FilePath, string> > >(); // Key = source, Value = tag HTML ConcurrentDictionary <FilePath, ConcurrentBag <string> > failures = new ConcurrentDictionary <FilePath, ConcurrentBag <string> >(); // Gather all links HtmlParser parser = new HtmlParser(); context.ParallelForEach(inputs, input => { GatherLinks(input, parser, links); }); // Perform validation Parallel.ForEach(links, link => { // Attempt to parse the URI Uri uri; if (!Uri.TryCreate(link.Key, UriKind.RelativeOrAbsolute, out uri)) { AddOrUpdateFailure(link.Value, failures); } // Adjustment for double-slash link prefix which means use http:// or https:// depending on current protocol // The Uri class treats these as relative, but they're really absolute if (uri.ToString().StartsWith("//") && !Uri.TryCreate($"http:{link.Key}", UriKind.Absolute, out uri)) { AddOrUpdateFailure(link.Value, failures); } // Relative if (!uri.IsAbsoluteUri && _validateRelativeLinks && !ValidateRelativeLink(uri, context)) { AddOrUpdateFailure(link.Value, failures); } // Absolute if (uri.IsAbsoluteUri && _validateAbsoluteLinks && !ValidateAbsoluteLink(uri, context)) { AddOrUpdateFailure(link.Value, failures); } }); // Report failures if (failures.Count > 0) { int failureCount = failures.Sum(x => x.Value.Count); string failureMessage = string.Join(Environment.NewLine, failures.Select(x => $"{x.Key.FullPath}{Environment.NewLine} - {string.Join(Environment.NewLine + " - ", x.Value)}")); Trace.TraceEvent( _asError ? TraceEventType.Error : TraceEventType.Warning, $"{failureCount} link validation failures:{Environment.NewLine}{failureMessage}"); } return(inputs); }
public void BaseSetUp() { TestTraceListener listener = new TestTraceListener(TestContext.CurrentContext.Test.ID); Trace.AddListener(listener); }
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context) { // Register all the MVC and Razor services // In the future, if DI is implemented for all Wyam, the IExecutionContext would be registered as a service // and the IHostingEnviornment would be registered as transient with the execution context provided in ctor IServiceCollection serviceCollection = new ServiceCollection(); IMvcCoreBuilder builder = serviceCollection .AddMvcCore() .AddRazorViewEngine(); builder.PartManager.FeatureProviders.Add(new MetadataReferenceFeatureProvider(context)); serviceCollection.Configure <RazorViewEngineOptions>(options => { options.ViewLocationExpanders.Add(new ViewLocationExpander()); }); serviceCollection .AddSingleton <ILoggerFactory, TraceLoggerFactory>() .AddSingleton <DiagnosticSource, SilentDiagnosticSource>() .AddSingleton <IHostingEnvironment, HostingEnvironment>() .AddSingleton <ObjectPoolProvider, DefaultObjectPoolProvider>() .AddSingleton <IExecutionContext>(context) .AddSingleton <IBasePageTypeProvider>(new BasePageTypeProvider(_basePageType ?? typeof(RazorPage))) .AddScoped <IMvcRazorHost, RazorHost>(); IServiceProvider services = serviceCollection.BuildServiceProvider(); // Eliminate input documents that we shouldn't process List <IDocument> validInputs = inputs .Where(x => _ignorePrefix == null || !x.ContainsKey(Keys.SourceFileName) || !x.FilePath(Keys.SourceFileName).FullPath.StartsWith(_ignorePrefix)) .ToList(); // Compile and evaluate the pages in parallel IServiceScopeFactory scopeFactory = services.GetRequiredService <IServiceScopeFactory>(); return(validInputs.AsParallel().Select(input => { Trace.Verbose("Compiling Razor for {0}", input.SourceString()); using (var scope = scopeFactory.CreateScope()) { // Get services IRazorViewEngine viewEngine = services.GetRequiredService <IRazorViewEngine>(); IRazorPageActivator pageActivator = services.GetRequiredService <IRazorPageActivator>(); HtmlEncoder htmlEncoder = services.GetRequiredService <HtmlEncoder>(); IRazorPageFactoryProvider pageFactoryProvider = services.GetRequiredService <IRazorPageFactoryProvider>(); IRazorCompilationService razorCompilationService = services.GetRequiredService <IRazorCompilationService>(); IHostingEnvironment hostingEnviornment = services.GetRequiredService <IHostingEnvironment>(); // Compile the view string relativePath = GetRelativePath(input, context); FilePath viewStartLocationPath = _viewStartPath?.Invoke <FilePath>(input, context); string viewStartLocation = viewStartLocationPath != null ? GetRelativePath(viewStartLocationPath, context) : null; string layoutLocation = _layoutPath?.Invoke <FilePath>(input, context)?.FullPath; IView view; using (Stream stream = input.GetStream()) { view = GetViewFromStream(relativePath, stream, viewStartLocation, layoutLocation, viewEngine, pageActivator, htmlEncoder, pageFactoryProvider, hostingEnviornment.WebRootFileProvider, razorCompilationService); } // Render the view Trace.Verbose("Processing Razor for {0}", input.SourceString()); using (StringWriter output = new StringWriter()) { Microsoft.AspNetCore.Mvc.Rendering.ViewContext viewContext = GetViewContext(scope.ServiceProvider, view, input, context, output); viewContext.View.RenderAsync(viewContext).GetAwaiter().GetResult(); return context.GetDocument(input, output.ToString()); } } })); }