internal void InstallPackages()
{
DirectoryPath packagesPath = GetAbsolutePackagesPath();
Trace.Information($"Installing packages to {packagesPath.FullPath} (using {(UseLocalPackagesFolder ? "local" : "global")} packages folder)");
try
{
// Add the global default sources if requested
if (UseGlobalPackageSources)
{
_sourceRepositories.AddGlobalDefaults();
}
// Get the local repository
SourceRepository localRepository = _sourceRepositories.CreateRepository(packagesPath.FullPath);
// Get the package manager and repositories
WyamFolderNuGetProject nuGetProject = new WyamFolderNuGetProject(_fileSystem, _assemblyLoader, _currentFramework, packagesPath.FullPath);
NuGetPackageManager packageManager = new NuGetPackageManager(_sourceRepositories, _settings, packagesPath.FullPath)
{
PackagesFolderNuGetProject = nuGetProject
};
IReadOnlyList <SourceRepository> remoteRepositories = _sourceRepositories.GetDefaultRepositories();
// Resolve all the versions
IReadOnlyList <SourceRepository> installationRepositories = remoteRepositories;
try
{
ResolveVersions(localRepository, remoteRepositories);
}
catch (Exception ex)
{
Trace.Verbose($"Exception while resolving package versions: {ex.Message}");
Trace.Warning("Error while resolving package versions, attempting without remote repositories");
installationRepositories = new[] { localRepository };
ResolveVersions(localRepository, Array.Empty <SourceRepository>());
}
// Install the packages (doing this synchronously since doing it in parallel triggers file lock errors in NuGet on a clean system)
try
{
InstallPackages(packageManager, installationRepositories);
}
catch (Exception ex)
{
Trace.Verbose($"Exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}");
Trace.Warning("Error while installing packages, attempting without remote repositories");
InstallPackages(packageManager, new[] { localRepository });
}
// Process the package (do this after all packages have been installed)
nuGetProject.ProcessAssembliesAndContent();
}
catch (Exception ex)
{
Trace.Verbose($"Unexpected exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}");
Trace.Warning("Error while installing packages, attempting to continue anyway");
}
}
// Internal for testing
internal void AddRecipePackageAndSetTheme()
{
if (Recipe == null && !string.IsNullOrEmpty(RecipeName))
{
KnownRecipe knownRecipe;
if (KnownRecipe.Values.TryGetValue(RecipeName, out knownRecipe))
{
Trace.Verbose($"Recipe {RecipeName} was in the lookup of known recipes");
// Make sure we're not ignoring packages
if (!IgnoreKnownRecipePackages)
{
// Add the package, but only if it wasn't added manually
if (!string.IsNullOrEmpty(knownRecipe.PackageId) && !PackageInstaller.ContainsPackage(knownRecipe.PackageId))
{
PackageInstaller.AddPackage(knownRecipe.PackageId, versionRange: $"[{Engine.Version}]", allowPrereleaseVersions: true);
}
}
else
{
Trace.Verbose("Ignoring known recipe packages");
}
// Set the theme if we don't already have one
if (string.IsNullOrEmpty(Theme))
{
Theme = knownRecipe.DefaultTheme;
}
}
else
{
Trace.Verbose($"Recipe {RecipeName} is not in the lookup of known recipes");
}
}
}
/// <summary>
/// Adds a package.
/// </summary>
/// <param name="packageId">The package identifier.</param>
/// <param name="packageSources">The package sources.</param>
/// <param name="versionRange">The version range.</param>
/// <param name="getLatest">If set to <c>true</c>, the latest version of the package will always be downloaded.</param>
/// <param name="allowPrereleaseVersions">If set to <c>true</c>, allow prerelease versions.</param>
/// <param name="allowUnlisted">If set to <c>true</c>, allow unlisted versions.</param>
/// <param name="exclusive">If set to <c>true</c>, only use the package sources defined for this package.</param>
public void AddPackage(
string packageId,
IEnumerable <string> packageSources = null,
string versionRange = null,
bool getLatest = false,
bool allowPrereleaseVersions = false,
bool allowUnlisted = false,
bool exclusive = false)
{
if (_packages.ContainsKey(packageId))
{
throw new ArgumentException($"A package with the ID {packageId} has already been added");
}
// If this package is a known Wyam extension and no version is specified, set to the current version
if (KnownExtension.Values.Values.Any(x => x.PackageId == packageId) && versionRange == null)
{
versionRange = $"[{Engine.Version}]";
Trace.Verbose($"Added known extension package {packageId} without version, setting version to {versionRange}");
}
_packages.Add(
packageId,
new Package(
_currentFramework,
packageId,
packageSources?.Select(_sourceRepositories.CreateRepository).ToList(),
versionRange,
getLatest,
allowPrereleaseVersions,
allowUnlisted,
exclusive));
}
public ExitCode Run(Preprocessor preprocessor)
{
if (GlobalArguments)
{
// Set verbose tracing
if (_verbose)
{
Trace.Level = System.Diagnostics.SourceLevels.Verbose;
}
// Attach
if (_attach)
{
Trace.Information($"Waiting for a debugger to attach to process {Process.GetCurrentProcess().Id} (or press a key to continue)...");
while (!Debugger.IsAttached && !Console.KeyAvailable)
{
Thread.Sleep(100);
}
if (Console.KeyAvailable)
{
Console.ReadKey(true);
Trace.Information("Key pressed, continuing execution");
}
else
{
Trace.Information("Debugger attached, continuing execution");
}
}
}
return(RunCommand(preprocessor));
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state,
Exception exception, Func <TState, Exception, string> formatter)
{
if (IsEnabled(logLevel))
{
Trace.TraceEvent(TraceMapping[logLevel], $"{_categoryName}: {formatter(state, exception)}");
}
}
public void BaseSetUp()
{
Directory.SetCurrentDirectory(TestContext.CurrentContext.TestDirectory);
TestTraceListener listener = new TestTraceListener(TestContext.CurrentContext.Test.ID);
_listeners.AddOrUpdate(TestContext.CurrentContext.Test.ID, listener, (x, y) => listener);
Trace.AddListener(Listener);
}
public void RemoveListener()
{
TestTraceListener listener = Listener;
if (listener != null)
{
Trace.RemoveListener(listener);
}
}
public void RemoveListener()
{
TestTraceListener listener = Trace.Listeners.OfType <TestTraceListener>()
.FirstOrDefault(x => x.TestId == TestContext.CurrentContext.Test.ID);
if (listener != null)
{
Trace.RemoveListener(listener);
}
}
private void CatalogClasses()
{
Stopwatch stopwatch = Stopwatch.StartNew();
using (Trace.WithIndent().Information("Cataloging classes"))
{
ClassCatalog.CatalogTypes(AssemblyLoader.DirectAssemblies);
stopwatch.Stop();
Trace.Information($"Classes cataloged in {stopwatch.ElapsedMilliseconds} ms");
}
}
private void InstallPackages(FilePath configFilePath)
{
Stopwatch stopwatch = Stopwatch.StartNew();
using (Trace.WithIndent().Information("Installing NuGet packages"))
{
PackageInstaller.InstallPackages(configFilePath);
stopwatch.Stop();
Trace.Information($"NuGet packages installed in {stopwatch.ElapsedMilliseconds} ms");
}
}
private void LoadAssemblies()
{
Stopwatch stopwatch = Stopwatch.StartNew();
using (Trace.WithIndent().Information("Loading assemblies"))
{
AssemblyLoader.Load();
stopwatch.Stop();
Trace.Information($"Assemblies loaded in {stopwatch.ElapsedMilliseconds} ms");
}
}
private static void UnhandledExceptionEvent(object sender, UnhandledExceptionEventArgs e)
{
// Exit with a error exit code
Exception exception = e.ExceptionObject as Exception;
if (exception != null)
{
Trace.Critical(exception.Message);
Trace.Verbose(exception.ToString());
}
Environment.Exit((int)ExitCode.UnhandledError);
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
// Expire the internal Razor cache if this is a new execution
// This needs to be done so that layouts/partials can be re-rendered if they've changed,
// otherwise Razor will just use the previously cached version of them
if (_executionId != Guid.Empty && _executionId != context.ExecutionId)
{
RazorService.ExpireChangeTokens();
}
_executionId = context.ExecutionId;
// Eliminate input documents that we shouldn't process
List <IDocument> validInputs = inputs
.Where(context, x => _ignorePrefix == null ||
!x.ContainsKey(Keys.SourceFileName) ||
!x.FilePath(Keys.SourceFileName).FullPath.StartsWith(_ignorePrefix))
.ToList();
if (validInputs.Count < inputs.Count)
{
Trace.Information($"Ignoring {inputs.Count - validInputs.Count} inputs due to source file name prefix");
}
// Compile and evaluate the pages in parallel
return(validInputs.AsParallel().Select(context, input =>
{
Trace.Verbose("Processing Razor for {0}", input.SourceString());
Stream contentStream = context.GetContentStream();
using (Stream inputStream = input.GetStream())
{
FilePath viewStartLocationPath = _viewStartPath?.Invoke <FilePath>(input, context);
RenderRequest request = new RenderRequest
{
Input = inputStream,
Output = contentStream,
BaseType = _basePageType,
Context = context,
Document = input,
LayoutLocation = _layoutPath?.Invoke <FilePath>(input, context)?.FullPath,
ViewStartLocation = viewStartLocationPath != null ? GetRelativePath(viewStartLocationPath, context) : null,
RelativePath = GetRelativePath(input, context),
Model = _model == null ? input : _model.Invoke(input, context),
};
RazorService.Render(request);
}
return context.GetDocument(input, contentStream);
}));
}
// Internal for testing
internal void AddThemePackagesAndPath()
{
string inputPath = Theme;
if (!string.IsNullOrEmpty(Theme))
{
KnownTheme knownTheme;
if (KnownTheme.Values.TryGetValue(Theme, out knownTheme))
{
Trace.Verbose($"Theme {Theme} was in the lookup of known themes");
inputPath = knownTheme.InputPath;
// Do a sanity check against the recipe (but only if we didn't explicitly specify one)
if (Recipe == null && !string.IsNullOrEmpty(RecipeName) && !string.IsNullOrEmpty(knownTheme.Recipe) &&
!string.Equals(RecipeName, knownTheme.Recipe, StringComparison.OrdinalIgnoreCase))
{
Trace.Warning($"Theme {Theme} is designed for recipe {knownTheme.Recipe} but is being used with recipe {RecipeName}, results may be unexpected");
}
// Make sure we're not ignoring theme packages
if (!IgnoreKnownThemePackages)
{
// Add any packages needed for the theme
if (knownTheme.PackageIds != null)
{
foreach (string themePackageId in knownTheme.PackageIds.Where(x => !PackageInstaller.ContainsPackage(x)))
{
PackageInstaller.AddPackage(themePackageId, allowPrereleaseVersions: true);
}
}
}
else
{
Trace.Verbose("Ignoring known theme packages");
}
}
else
{
Trace.Verbose($"Theme {Theme} is not in the lookup of known themes, assuming it's an input path");
}
}
// Insert the theme path
if (!string.IsNullOrEmpty(inputPath))
{
_engine.FileSystem.InputPaths.Insert(0, new DirectoryPath(inputPath));
}
}
private int Run(string[] args)
{
// Add a default trace listener
Trace.AddListener(new SimpleColorConsoleTraceListener {
TraceOutputOptions = System.Diagnostics.TraceOptions.None
});
// Output version info
Trace.Information($"Wyam version {Engine.Version}");
// It's not a serious console app unless there's some ASCII art
OutputLogo();
// Make sure we're not running under Mono
if (Type.GetType("Mono.Runtime") != null)
{
Trace.Critical("The Mono runtime is not supported. Please check the GitHub repository and issue tracker for information on .NET Core support for cross platform execution.");
return((int)ExitCode.UnsupportedRuntime);
}
// Parse the command line
Preprocessor preprocessor = new Preprocessor();
Command command;
try
{
bool hasParseArgsErrors;
command = CommandParser.Parse(args, preprocessor, out hasParseArgsErrors);
if (command == null)
{
return(hasParseArgsErrors ? (int)ExitCode.CommandLineError : (int)ExitCode.Normal);
}
}
catch (Exception ex)
{
Trace.Error("Error while parsing command line: {0}", ex.Message);
if (Trace.Level == SourceLevels.Verbose)
{
Trace.Error("Stack trace:{0}{1}", Environment.NewLine, ex.StackTrace);
}
return((int)ExitCode.CommandLineError);
}
// Run the command
return((int)command.Run(preprocessor));
}
private void Evaluate(string code)
{
if (string.IsNullOrEmpty(code))
{
return;
}
Stopwatch stopwatch = Stopwatch.StartNew();
using (Trace.WithIndent().Information("Evaluating configuration script"))
{
CacheManager cacheManager = new CacheManager(_engine, _scriptManager, ConfigDllPath, ConfigHashPath, OutputScriptPath);
cacheManager.EvaluateCode(code, ClassCatalog.GetClasses <IModule>().ToList(), OutputScript, IgnoreConfigHash, NoOutputConfigAssembly);
stopwatch.Stop();
Trace.Information($"Evaluated configuration script in {stopwatch.ElapsedMilliseconds} ms");
}
}
private int Run(string[] args)
{
// Add a default trace listener
Trace.AddListener(new SimpleColorConsoleTraceListener {
TraceOutputOptions = System.Diagnostics.TraceOptions.None
});
// Output version info
AssemblyInformationalVersionAttribute versionAttribute
= Attribute.GetCustomAttribute(typeof(Program).Assembly, typeof(AssemblyInformationalVersionAttribute)) as AssemblyInformationalVersionAttribute;
Trace.Information("Wyam version {0}", versionAttribute == null ? "unknown" : versionAttribute.InformationalVersion);
// It's not a serious console app unless there's some ASCII art
OutputLogo();
// Parse the command line
Preprocessor preprocessor = new Preprocessor();
Command command;
try
{
bool hasParseArgsErrors;
command = CommandParser.Parse(args, preprocessor, out hasParseArgsErrors);
if (command == null)
{
return(hasParseArgsErrors ? (int)ExitCode.CommandLineError : (int)ExitCode.Normal);
}
}
catch (Exception ex)
{
Trace.Error("Error while parsing command line: {0}", ex.Message);
if (Trace.Level == SourceLevels.Verbose)
{
Trace.Error("Stack trace:{0}{1}", Environment.NewLine, ex.StackTrace);
}
return((int)ExitCode.CommandLineError);
}
// Run the command
return((int)command.Run(preprocessor));
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
#pragma warning disable RCS1163 // Unused parameter.
// Handle invalid HTTPS certificates and allow alternate security protocols (see http://stackoverflow.com/a/5670954/807064)
ServicePointManager.ServerCertificateValidationCallback = (s, cert, chain, ssl) => true;
#pragma warning restore RCS1163 // Unused parameter.
// Key = link, Value = source, tag HTML
ConcurrentDictionary <string, ConcurrentBag <(string documentSource, string outerHtml)> > links =
new ConcurrentDictionary <string, ConcurrentBag <(string documentSource, string outerHtml)> >();
// Key = source, Value = tag HTML
ConcurrentDictionary <string, ConcurrentBag <string> > failures = new ConcurrentDictionary <string, ConcurrentBag <string> >();
// Gather all links
HtmlParser parser = new HtmlParser();
context.ParallelForEach(inputs, input => GatherLinks(input, parser, links));
// This policy will limit the number of executing link validations.
// Limit the amount of concurrent link checks to avoid overwhelming servers.
Task[] tasks = links.Select(
async link =>
{
// Attempt to parse the URI
if (!Uri.TryCreate(link.Key, UriKind.RelativeOrAbsolute, out Uri uri))
{
AddOrUpdateFailure(link.Value, failures);
}
// Adjustment for double-slash link prefix which means use http:// or https:// depending on current protocol
// The Uri class treats these as relative, but they're really absolute
if (uri.ToString().StartsWith("//") && !Uri.TryCreate($"http:{link.Key}", UriKind.Absolute, out uri))
{
AddOrUpdateFailure(link.Value, failures);
}
// Relative
if (!uri.IsAbsoluteUri && _validateRelativeLinks && !(await ValidateRelativeLink(uri, context).ConfigureAwait(false)))
{
AddOrUpdateFailure(link.Value, failures);
}
// Absolute
if (uri.IsAbsoluteUri && _validateAbsoluteLinks && !(await ValidateAbsoluteLink(uri, context).ConfigureAwait(false)))
{
AddOrUpdateFailure(link.Value, failures);
}
}).ToArray();
Task.WaitAll(tasks);
// Report failures
if (failures.Count > 0)
{
int failureCount = failures.Sum(x => x.Value.Count);
string failureMessage = string.Join(
Environment.NewLine,
failures.Select(x => $"{x.Key}{Environment.NewLine} - {string.Join(Environment.NewLine + " - ", x.Value)}"));
Trace.TraceEvent(
_asError ? TraceEventType.Error : TraceEventType.Warning,
$"{failureCount} link validation failures:{Environment.NewLine}{failureMessage}");
}
return(inputs);
}
protected override ExitCode RunCommand(Preprocessor preprocessor)
{
// Get the standard input stream
_configOptions.Stdin = StandardInputReader.Read();
// Fix the root folder and other files
DirectoryPath currentDirectory = Environment.CurrentDirectory;
_configOptions.RootPath = _configOptions.RootPath == null ? currentDirectory : currentDirectory.Combine(_configOptions.RootPath);
_logFilePath = _logFilePath == null ? null : _configOptions.RootPath.CombineFile(_logFilePath);
_configOptions.ConfigFilePath = _configOptions.RootPath.CombineFile(_configOptions.ConfigFilePath ?? "config.wyam");
// Set up the log file
if (_logFilePath != null)
{
Trace.AddListener(new SimpleFileTraceListener(_logFilePath.FullPath));
}
// Get the engine and configurator
EngineManager engineManager = EngineManager.Get(preprocessor, _configOptions);
if (engineManager == null)
{
return(ExitCode.CommandLineError);
}
// Configure and execute
if (!engineManager.Configure())
{
return(ExitCode.ConfigurationError);
}
if (_verifyConfig)
{
Trace.Information("No errors. Exiting.");
return(ExitCode.Normal);
}
Trace.Information($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.RootPath}");
Trace.Information($"Input path(s):{Environment.NewLine} {string.Join(Environment.NewLine + " ", engineManager.Engine.FileSystem.InputPaths)}");
Trace.Information($"Output path:{Environment.NewLine} {engineManager.Engine.FileSystem.OutputPath}");
if (!engineManager.Execute())
{
return(ExitCode.ExecutionError);
}
bool messagePump = false;
// Start the preview server
IDisposable previewServer = null;
if (_preview)
{
messagePump = true;
DirectoryPath previewPath = _previewRoot == null
? engineManager.Engine.FileSystem.GetOutputDirectory().Path
: engineManager.Engine.FileSystem.GetOutputDirectory(_previewRoot).Path;
previewServer = PreviewServer.Start(previewPath, _previewPort, _previewForceExtension);
}
// Start the watchers
IDisposable inputFolderWatcher = null;
IDisposable configFileWatcher = null;
if (_watch)
{
messagePump = true;
Trace.Information("Watching paths(s) {0}", string.Join(", ", engineManager.Engine.FileSystem.InputPaths));
inputFolderWatcher = new ActionFileSystemWatcher(engineManager.Engine.FileSystem.GetOutputDirectory().Path,
engineManager.Engine.FileSystem.GetInputDirectories().Select(x => x.Path), true, "*.*", path =>
{
_changedFiles.Enqueue(path);
_messageEvent.Set();
});
if (_configOptions.ConfigFilePath != null)
{
Trace.Information("Watching configuration file {0}", _configOptions.ConfigFilePath);
configFileWatcher = new ActionFileSystemWatcher(engineManager.Engine.FileSystem.GetOutputDirectory().Path,
new[] { _configOptions.ConfigFilePath.Directory }, false, _configOptions.ConfigFilePath.FileName.FullPath, path =>
{
FilePath filePath = new FilePath(path);
if (_configOptions.ConfigFilePath.Equals(filePath))
{
_newEngine.Set();
_messageEvent.Set();
}
});
}
}
// Start the message pump if an async process is running
ExitCode exitCode = ExitCode.Normal;
if (messagePump)
{
// Only wait for a key if console input has not been redirected, otherwise it's on the caller to exit
if (!Console.IsInputRedirected)
{
// Start the key listening thread
var thread = new Thread(() =>
{
Trace.Information("Hit any key to exit");
Console.ReadKey();
_exit.Set();
_messageEvent.Set();
})
{
IsBackground = true
};
thread.Start();
}
// Wait for activity
while (true)
{
_messageEvent.WaitOne(); // Blocks the current thread until a signal
if (_exit)
{
break;
}
// See if we need a new engine
if (_newEngine)
{
// Get a new engine
Trace.Information("Configuration file {0} has changed, re-running", _configOptions.ConfigFilePath);
engineManager.Dispose();
engineManager = EngineManager.Get(preprocessor, _configOptions);
// Configure and execute
if (!engineManager.Configure())
{
exitCode = ExitCode.ConfigurationError;
break;
}
Console.WriteLine($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.RootPath}");
Console.WriteLine($"Input path(s):{Environment.NewLine} {string.Join(Environment.NewLine + " ", engineManager.Engine.FileSystem.InputPaths)}");
Console.WriteLine($"Root path:{Environment.NewLine} {engineManager.Engine.FileSystem.OutputPath}");
if (!engineManager.Execute())
{
exitCode = ExitCode.ExecutionError;
break;
}
// Clear the changed files since we just re-ran
string changedFile;
while (_changedFiles.TryDequeue(out changedFile))
{
}
_newEngine.Unset();
}
else
{
// Execute if files have changed
HashSet <string> changedFiles = new HashSet <string>();
string changedFile;
while (_changedFiles.TryDequeue(out changedFile))
{
if (changedFiles.Add(changedFile))
{
Trace.Verbose("{0} has changed", changedFile);
}
}
if (changedFiles.Count > 0)
{
Trace.Information("{0} files have changed, re-executing", changedFiles.Count);
if (!engineManager.Execute())
{
exitCode = ExitCode.ExecutionError;
break;
}
}
}
// Check one more time for exit
if (_exit)
{
break;
}
Trace.Information("Hit any key to exit");
_messageEvent.Reset();
}
// Shutdown
Trace.Information("Shutting down");
engineManager.Dispose();
inputFolderWatcher?.Dispose();
configFileWatcher?.Dispose();
previewServer?.Dispose();
}
return(exitCode);
}
protected override void ParseOptions(ArgumentSyntax syntax)
{
syntax.DefineOption("w|watch", ref _watch, "Watches the input folder for any changes.");
_preview = syntax.DefineOption("p|preview", ref _previewPort, false, "Start the preview web server on the specified port (default is " + _previewPort + ").").IsSpecified;
if (syntax.DefineOption("force-ext", ref _previewForceExtension, "Force the use of extensions in the preview web server (by default, extensionless URLs may be used).").IsSpecified&& !_preview)
{
syntax.ReportError("force-ext can only be specified if the preview server is running.");
}
if (syntax.DefineOption("virtual-dir", ref _previewVirtualDirectory, DirectoryPath.FromString, "Serve files in the preview web server under the specified virtual directory.").IsSpecified&& !_preview)
{
syntax.ReportError("virtual-dir can only be specified if the preview server is running.");
}
if (syntax.DefineOption("preview-root", ref _previewRoot, DirectoryPath.FromString, "The path to the root of the preview server, if not the output folder.").IsSpecified&& !_preview)
{
syntax.ReportError("preview-root can only be specified if the preview server is running.");
}
syntax.DefineOptionList("i|input", ref _configOptions.InputPaths, DirectoryPath.FromString, "The path(s) of input files, can be absolute or relative to the current folder.");
syntax.DefineOption("o|output", ref _configOptions.OutputPath, DirectoryPath.FromString, "The path to output files, can be absolute or relative to the current folder.");
syntax.DefineOption("c|config", ref _configOptions.ConfigFilePath, FilePath.FromString, "Configuration file (by default, config.wyam is used).");
syntax.DefineOption("u|update-packages", ref _configOptions.UpdatePackages, "Check the NuGet server for more recent versions of each package and update them if applicable.");
syntax.DefineOption("use-local-packages", ref _configOptions.UseLocalPackages, "Toggles the use of a local NuGet packages folder.");
syntax.DefineOption("use-global-sources", ref _configOptions.UseGlobalSources, "Toggles the use of the global NuGet sources (default is false).");
syntax.DefineOption("packages-path", ref _configOptions.PackagesPath, DirectoryPath.FromString, "The packages path to use (only if use-local is true).");
syntax.DefineOption("output-script", ref _configOptions.OutputScript, "Outputs the config script after it's been processed for further debugging.");
syntax.DefineOption("verify-config", ref _verifyConfig, false, "Compile the configuration but do not execute.");
syntax.DefineOption("noclean", ref _configOptions.NoClean, "Prevents cleaning of the output path on each execution.");
syntax.DefineOption("nocache", ref _configOptions.NoCache, "Prevents caching information during execution (less memory usage but slower execution).");
_logFilePath = $"wyam-{DateTime.Now:yyyyMMddHHmmssfff}.txt";
if (!syntax.DefineOption("l|log", ref _logFilePath, FilePath.FromString, false, "Log all trace messages to the specified log file (by default, wyam-[datetime].txt).").IsSpecified)
{
_logFilePath = null;
}
// Metadata
// TODO: Remove this dictionary and initial/global options
Dictionary <string, object> settingsDictionary = new Dictionary <string, object>();
IReadOnlyList <string> globalMetadata = null;
if (syntax.DefineOptionList("g|global", ref globalMetadata, "Deprecated, do not use.").IsSpecified)
{
Trace.Warning("-g/--global is deprecated and will be removed in a future version. Please use -s/--setting instead.");
AddSettings(settingsDictionary, globalMetadata);
}
IReadOnlyList <string> initialMetadata = null;
if (syntax.DefineOptionList("initial", ref initialMetadata, "Deprecated, do not use.").IsSpecified)
{
Trace.Warning("--initial is deprecated and will be removed in a future version. Please use -s/--setting instead.");
AddSettings(settingsDictionary, initialMetadata);
}
IReadOnlyList <string> settings = null;
if (syntax.DefineOptionList("s|setting", ref settings, "Specifies a setting as a key=value pair. Use the syntax [x,y] to specify an array value.").IsSpecified)
{
// _configOptions.Settings = MetadataParser.Parse(settings); TODO: Use this when AddSettings() is removed
AddSettings(settingsDictionary, settings);
}
if (settingsDictionary.Count > 0)
{
_configOptions.Settings = settingsDictionary;
}
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
HtmlParser parser = new HtmlParser();
using (IJsEnginePool enginePool = context.GetJsEnginePool(x =>
{
if (string.IsNullOrWhiteSpace(_highlightJsFile))
{
x.ExecuteResource("highlight-all.js", typeof(Highlight));
}
else
{
x.ExecuteFile(_highlightJsFile);
}
}))
{
return(inputs.AsParallel().Select(context, input =>
{
// We materialize the list before exiting the using statement, so safe to access enginePool
// ReSharper disable once AccessToDisposedClosure
using (IJsEngine engine = enginePool.GetEngine())
{
try
{
using (Stream stream = input.GetStream())
using (IHtmlDocument htmlDocument = parser.Parse(stream))
{
foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector))
{
// Don't highlight anything that potentially is already highlighted
if (element.ClassList.Contains("hljs"))
{
continue;
}
// Make sure to use TextContent, otherwise you'll get escaped html which highlight.js won't parse
engine.SetVariableValue("input", element.TextContent);
// Check if they specified a language in their code block
string language = element.ClassList.FirstOrDefault(i => i.StartsWith("language"));
try
{
if (language != null)
{
engine.SetVariableValue("language", language.Replace("language-", ""));
engine.Execute("result = hljs.highlight(language, input)");
}
else
{
language = "(auto)"; // set this to auto in case there is an exception below
engine.Execute("result = hljs.highlightAuto(input)");
string detectedLanguage = engine.Evaluate <string>("result.language");
if (string.IsNullOrWhiteSpace(detectedLanguage) == false)
{
element.ClassList.Add("language-" + detectedLanguage);
}
}
element.ClassList.Add("hljs");
string formatted = engine.Evaluate <string>("result.value");
element.InnerHtml = formatted;
}
catch (Exception innerEx)
{
if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage)
{
Trace.Warning("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message);
}
else
{
Trace.Information("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message);
}
}
}
string content = htmlDocument.ToHtml();
return context.GetDocument(input, content);
}
}
catch (Exception ex)
{
Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message);
return input;
}
}
}).ToList());
}
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
HtmlParser parser = new HtmlParser();
using (IJavaScriptEnginePool enginePool = context.GetJavaScriptEnginePool(x =>
{
if (string.IsNullOrWhiteSpace(_highlightJsFile))
{
x.ExecuteResource("highlight-all.js", typeof(Highlight));
}
else
{
x.ExecuteFile(_highlightJsFile);
}
}))
{
return(inputs.AsParallel().Select(context, input =>
{
try
{
using (Stream stream = input.GetStream())
{
using (IHtmlDocument htmlDocument = parser.Parse(stream))
{
foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector))
{
// Don't highlight anything that potentially is already highlighted
if (element.ClassList.Contains("hljs"))
{
continue;
}
try
{
HighlightElement(enginePool, element);
}
catch (Exception innerEx)
{
if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage)
{
Trace.Warning($"Exception while highlighting source code: {innerEx.Message}");
}
else
{
Trace.Information($"Exception while highlighting source code: {innerEx.Message}");
}
}
}
Stream contentStream = context.GetContentStream();
using (StreamWriter writer = contentStream.GetWriter())
{
htmlDocument.ToHtml(writer, HtmlMarkupFormatter.Instance);
writer.Flush();
return context.GetDocument(input, contentStream);
}
}
}
}
catch (Exception ex)
{
Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message);
return input;
}
}).ToList());
}
}
// Internal for testing
internal static bool ValidateRelativeLink(Uri uri, IExecutionContext context)
{
List <FilePath> checkPaths = new List <FilePath>();
// Remove the query string and fragment, if any
string normalizedPath = uri.ToString();
if (normalizedPath.Contains("#"))
{
normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("#", StringComparison.Ordinal));
}
if (normalizedPath.Contains("?"))
{
normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("?", StringComparison.Ordinal));
}
if (normalizedPath == string.Empty)
{
return(true);
}
// Remove the link root if there is one and remove the preceding slash
if (context.Settings.DirectoryPath(Keys.LinkRoot) != null &&
normalizedPath.StartsWith(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath))
{
normalizedPath = normalizedPath.Substring(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath.Length);
}
if (normalizedPath.StartsWith("/"))
{
normalizedPath = normalizedPath.Length > 1 ? normalizedPath.Substring(1) : string.Empty;
}
// Add the base path
if (normalizedPath != string.Empty)
{
checkPaths.Add(new FilePath(normalizedPath));
}
// Add filenames
checkPaths.AddRange(LinkGenerator.DefaultHidePages.Select(x => new FilePath(normalizedPath == string.Empty ? x : $"{normalizedPath}/{x}")));
// Add extensions
checkPaths.AddRange(LinkGenerator.DefaultHideExtensions.SelectMany(x => checkPaths.Select(y => y.AppendExtension(x))).ToArray());
// Check all the candidate paths
FilePath validatedPath = checkPaths.FirstOrDefault(x =>
{
IFile outputFile;
try
{
outputFile = context.FileSystem.GetOutputFile(x);
}
catch (Exception ex)
{
Trace.Warning($"Could not validate path {x.FullPath} for relative link {uri}: {ex.Message}");
return(false);
}
return(outputFile.Exists);
});
if (validatedPath != null)
{
Trace.Verbose($"Validated relative link {uri} at {validatedPath.FullPath}");
return(true);
}
Trace.Warning($"Validation failure for relative link {uri}: could not find output file at any of {string.Join(", ", checkPaths.Select(x => x.FullPath))}");
return(false);
}
// Internal for testing
internal static bool ValidateAbsoluteLink(Uri uri, IExecutionContext context)
{
// Create a request
HttpWebRequest request;
try
{
request = WebRequest.Create(uri) as HttpWebRequest;
}
catch (NotSupportedException ex)
{
Trace.Warning($"Skipping absolute link {uri}: {ex.Message}");
return(true);
}
if (request == null)
{
Trace.Warning($"Skipping absolute link {uri}: only HTTP/HTTPS links are validated");
return(true);
}
// Set request properties
request.Timeout = 60000; // 60 seconds
// Perform request as HEAD
HttpWebResponse response;
request.Method = "HEAD";
try
{
response = (HttpWebResponse)request.GetResponse();
response.Close();
}
catch (WebException)
{
response = null;
}
// Check the status code
if (response != null)
{
if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400)
{
Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}");
return(true);
}
}
// Try one more time as GET
request.Method = "GET";
try
{
response = (HttpWebResponse)request.GetResponse();
response.Close();
}
catch (WebException ex)
{
Trace.Warning($"Validation failure for absolute link {uri}: {ex.Message}");
return(false);
}
// Check the status code
if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400)
{
Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}");
return(true);
}
Trace.Warning($"Validation failure for absolute link {uri}: returned status code {(int)response.StatusCode} {response.StatusCode}");
return(false);
}
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
// Handle invalid HTTPS certificates and allow alternate security protocols (see http://stackoverflow.com/a/5670954/807064)
ServicePointManager.ServerCertificateValidationCallback = (s, cert, chain, ssl) => true;
// Key = link, Value = source, tag HTML
ConcurrentDictionary <string, ConcurrentBag <Tuple <FilePath, string> > > links =
new ConcurrentDictionary <string, ConcurrentBag <Tuple <FilePath, string> > >();
// Key = source, Value = tag HTML
ConcurrentDictionary <FilePath, ConcurrentBag <string> > failures = new ConcurrentDictionary <FilePath, ConcurrentBag <string> >();
// Gather all links
HtmlParser parser = new HtmlParser();
context.ParallelForEach(inputs, input =>
{
GatherLinks(input, parser, links);
});
// Perform validation
Parallel.ForEach(links, link =>
{
// Attempt to parse the URI
Uri uri;
if (!Uri.TryCreate(link.Key, UriKind.RelativeOrAbsolute, out uri))
{
AddOrUpdateFailure(link.Value, failures);
}
// Adjustment for double-slash link prefix which means use http:// or https:// depending on current protocol
// The Uri class treats these as relative, but they're really absolute
if (uri.ToString().StartsWith("//") && !Uri.TryCreate($"http:{link.Key}", UriKind.Absolute, out uri))
{
AddOrUpdateFailure(link.Value, failures);
}
// Relative
if (!uri.IsAbsoluteUri && _validateRelativeLinks && !ValidateRelativeLink(uri, context))
{
AddOrUpdateFailure(link.Value, failures);
}
// Absolute
if (uri.IsAbsoluteUri && _validateAbsoluteLinks && !ValidateAbsoluteLink(uri, context))
{
AddOrUpdateFailure(link.Value, failures);
}
});
// Report failures
if (failures.Count > 0)
{
int failureCount = failures.Sum(x => x.Value.Count);
string failureMessage = string.Join(Environment.NewLine,
failures.Select(x => $"{x.Key.FullPath}{Environment.NewLine} - {string.Join(Environment.NewLine + " - ", x.Value)}"));
Trace.TraceEvent(
_asError ? TraceEventType.Error : TraceEventType.Warning,
$"{failureCount} link validation failures:{Environment.NewLine}{failureMessage}");
}
return(inputs);
}
public void BaseSetUp()
{
TestTraceListener listener = new TestTraceListener(TestContext.CurrentContext.Test.ID);
Trace.AddListener(listener);
}
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
// Register all the MVC and Razor services
// In the future, if DI is implemented for all Wyam, the IExecutionContext would be registered as a service
// and the IHostingEnviornment would be registered as transient with the execution context provided in ctor
IServiceCollection serviceCollection = new ServiceCollection();
IMvcCoreBuilder builder = serviceCollection
.AddMvcCore()
.AddRazorViewEngine();
builder.PartManager.FeatureProviders.Add(new MetadataReferenceFeatureProvider(context));
serviceCollection.Configure <RazorViewEngineOptions>(options =>
{
options.ViewLocationExpanders.Add(new ViewLocationExpander());
});
serviceCollection
.AddSingleton <ILoggerFactory, TraceLoggerFactory>()
.AddSingleton <DiagnosticSource, SilentDiagnosticSource>()
.AddSingleton <IHostingEnvironment, HostingEnvironment>()
.AddSingleton <ObjectPoolProvider, DefaultObjectPoolProvider>()
.AddSingleton <IExecutionContext>(context)
.AddSingleton <IBasePageTypeProvider>(new BasePageTypeProvider(_basePageType ?? typeof(RazorPage)))
.AddScoped <IMvcRazorHost, RazorHost>();
IServiceProvider services = serviceCollection.BuildServiceProvider();
// Eliminate input documents that we shouldn't process
List <IDocument> validInputs = inputs
.Where(x => _ignorePrefix == null ||
!x.ContainsKey(Keys.SourceFileName) ||
!x.FilePath(Keys.SourceFileName).FullPath.StartsWith(_ignorePrefix))
.ToList();
// Compile and evaluate the pages in parallel
IServiceScopeFactory scopeFactory = services.GetRequiredService <IServiceScopeFactory>();
return(validInputs.AsParallel().Select(input =>
{
Trace.Verbose("Compiling Razor for {0}", input.SourceString());
using (var scope = scopeFactory.CreateScope())
{
// Get services
IRazorViewEngine viewEngine = services.GetRequiredService <IRazorViewEngine>();
IRazorPageActivator pageActivator = services.GetRequiredService <IRazorPageActivator>();
HtmlEncoder htmlEncoder = services.GetRequiredService <HtmlEncoder>();
IRazorPageFactoryProvider pageFactoryProvider = services.GetRequiredService <IRazorPageFactoryProvider>();
IRazorCompilationService razorCompilationService = services.GetRequiredService <IRazorCompilationService>();
IHostingEnvironment hostingEnviornment = services.GetRequiredService <IHostingEnvironment>();
// Compile the view
string relativePath = GetRelativePath(input, context);
FilePath viewStartLocationPath = _viewStartPath?.Invoke <FilePath>(input, context);
string viewStartLocation = viewStartLocationPath != null ? GetRelativePath(viewStartLocationPath, context) : null;
string layoutLocation = _layoutPath?.Invoke <FilePath>(input, context)?.FullPath;
IView view;
using (Stream stream = input.GetStream())
{
view = GetViewFromStream(relativePath, stream, viewStartLocation, layoutLocation, viewEngine, pageActivator,
htmlEncoder, pageFactoryProvider, hostingEnviornment.WebRootFileProvider, razorCompilationService);
}
// Render the view
Trace.Verbose("Processing Razor for {0}", input.SourceString());
using (StringWriter output = new StringWriter())
{
Microsoft.AspNetCore.Mvc.Rendering.ViewContext viewContext =
GetViewContext(scope.ServiceProvider, view, input, context, output);
viewContext.View.RenderAsync(viewContext).GetAwaiter().GetResult();
return context.GetDocument(input, output.ToString());
}
}
}));
}
