internal void InstallPackages()
{
DirectoryPath packagesPath = GetAbsolutePackagesPath();
Trace.Information($"Installing packages to {packagesPath.FullPath} (using {(UseLocalPackagesFolder ? "local" : "global")} packages folder)");
try
{
// Add the global default sources if requested
if (UseGlobalPackageSources)
{
_sourceRepositories.AddGlobalDefaults();
}
// Get the local repository
SourceRepository localRepository = _sourceRepositories.CreateRepository(packagesPath.FullPath);
// Get the package manager and repositories
WyamFolderNuGetProject nuGetProject = new WyamFolderNuGetProject(_fileSystem, _assemblyLoader, _currentFramework, packagesPath.FullPath);
NuGetPackageManager packageManager = new NuGetPackageManager(_sourceRepositories, _settings, packagesPath.FullPath)
{
PackagesFolderNuGetProject = nuGetProject
};
IReadOnlyList <SourceRepository> remoteRepositories = _sourceRepositories.GetDefaultRepositories();
// Resolve all the versions
IReadOnlyList <SourceRepository> installationRepositories = remoteRepositories;
try
{
ResolveVersions(localRepository, remoteRepositories);
}
catch (Exception ex)
{
Trace.Verbose($"Exception while resolving package versions: {ex.Message}");
Trace.Warning("Error while resolving package versions, attempting without remote repositories");
installationRepositories = new[] { localRepository };
ResolveVersions(localRepository, Array.Empty <SourceRepository>());
}
// Install the packages (doing this synchronously since doing it in parallel triggers file lock errors in NuGet on a clean system)
try
{
InstallPackages(packageManager, installationRepositories);
}
catch (Exception ex)
{
Trace.Verbose($"Exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}");
Trace.Warning("Error while installing packages, attempting without remote repositories");
InstallPackages(packageManager, new[] { localRepository });
}
// Process the package (do this after all packages have been installed)
nuGetProject.ProcessAssembliesAndContent();
}
catch (Exception ex)
{
Trace.Verbose($"Unexpected exception while installing packages: {(ex is AggregateException ? string.Join("; ", ((AggregateException)ex).InnerExceptions.Select(x => x.Message)) : ex.Message)}");
Trace.Warning("Error while installing packages, attempting to continue anyway");
}
}
// Internal for testing
internal void AddThemePackagesAndPath()
{
string inputPath = Theme;
if (!string.IsNullOrEmpty(Theme))
{
KnownTheme knownTheme;
if (KnownTheme.Values.TryGetValue(Theme, out knownTheme))
{
Trace.Verbose($"Theme {Theme} was in the lookup of known themes");
inputPath = knownTheme.InputPath;
// Do a sanity check against the recipe (but only if we didn't explicitly specify one)
if (Recipe == null && !string.IsNullOrEmpty(RecipeName) && !string.IsNullOrEmpty(knownTheme.Recipe) &&
!string.Equals(RecipeName, knownTheme.Recipe, StringComparison.OrdinalIgnoreCase))
{
Trace.Warning($"Theme {Theme} is designed for recipe {knownTheme.Recipe} but is being used with recipe {RecipeName}, results may be unexpected");
}
// Make sure we're not ignoring theme packages
if (!IgnoreKnownThemePackages)
{
// Add any packages needed for the theme
if (knownTheme.PackageIds != null)
{
foreach (string themePackageId in knownTheme.PackageIds.Where(x => !PackageInstaller.ContainsPackage(x)))
{
PackageInstaller.AddPackage(themePackageId, allowPrereleaseVersions: true);
}
}
}
else
{
Trace.Verbose("Ignoring known theme packages");
}
}
else
{
Trace.Verbose($"Theme {Theme} is not in the lookup of known themes, assuming it's an input path");
}
}
// Insert the theme path
if (!string.IsNullOrEmpty(inputPath))
{
_engine.FileSystem.InputPaths.Insert(0, new DirectoryPath(inputPath));
}
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
HtmlParser parser = new HtmlParser();
using (IJsEnginePool enginePool = context.GetJsEnginePool(x =>
{
if (string.IsNullOrWhiteSpace(_highlightJsFile))
{
x.ExecuteResource("highlight-all.js", typeof(Highlight));
}
else
{
x.ExecuteFile(_highlightJsFile);
}
}))
{
return(inputs.AsParallel().Select(context, input =>
{
// We materialize the list before exiting the using statement, so safe to access enginePool
// ReSharper disable once AccessToDisposedClosure
using (IJsEngine engine = enginePool.GetEngine())
{
try
{
using (Stream stream = input.GetStream())
using (IHtmlDocument htmlDocument = parser.Parse(stream))
{
foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector))
{
// Don't highlight anything that potentially is already highlighted
if (element.ClassList.Contains("hljs"))
{
continue;
}
// Make sure to use TextContent, otherwise you'll get escaped html which highlight.js won't parse
engine.SetVariableValue("input", element.TextContent);
// Check if they specified a language in their code block
string language = element.ClassList.FirstOrDefault(i => i.StartsWith("language"));
try
{
if (language != null)
{
engine.SetVariableValue("language", language.Replace("language-", ""));
engine.Execute("result = hljs.highlight(language, input)");
}
else
{
language = "(auto)"; // set this to auto in case there is an exception below
engine.Execute("result = hljs.highlightAuto(input)");
string detectedLanguage = engine.Evaluate <string>("result.language");
if (string.IsNullOrWhiteSpace(detectedLanguage) == false)
{
element.ClassList.Add("language-" + detectedLanguage);
}
}
element.ClassList.Add("hljs");
string formatted = engine.Evaluate <string>("result.value");
element.InnerHtml = formatted;
}
catch (Exception innerEx)
{
if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage)
{
Trace.Warning("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message);
}
else
{
Trace.Information("Exception while highlighting source code for {0} using language {1}: {2}", input.SourceString(), language, innerEx.Message);
}
}
}
string content = htmlDocument.ToHtml();
return context.GetDocument(input, content);
}
}
catch (Exception ex)
{
Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message);
return input;
}
}
}).ToList());
}
}
// Internal for testing
internal static bool ValidateAbsoluteLink(Uri uri, IExecutionContext context)
{
// Create a request
HttpWebRequest request;
try
{
request = WebRequest.Create(uri) as HttpWebRequest;
}
catch (NotSupportedException ex)
{
Trace.Warning($"Skipping absolute link {uri}: {ex.Message}");
return(true);
}
if (request == null)
{
Trace.Warning($"Skipping absolute link {uri}: only HTTP/HTTPS links are validated");
return(true);
}
// Set request properties
request.Timeout = 60000; // 60 seconds
// Perform request as HEAD
HttpWebResponse response;
request.Method = "HEAD";
try
{
response = (HttpWebResponse)request.GetResponse();
response.Close();
}
catch (WebException)
{
response = null;
}
// Check the status code
if (response != null)
{
if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400)
{
Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}");
return(true);
}
}
// Try one more time as GET
request.Method = "GET";
try
{
response = (HttpWebResponse)request.GetResponse();
response.Close();
}
catch (WebException ex)
{
Trace.Warning($"Validation failure for absolute link {uri}: {ex.Message}");
return(false);
}
// Check the status code
if ((int)response.StatusCode >= 100 && (int)response.StatusCode < 400)
{
Trace.Verbose($"Validated absolute link {uri} with status code {(int)response.StatusCode} {response.StatusCode}");
return(true);
}
Trace.Warning($"Validation failure for absolute link {uri}: returned status code {(int)response.StatusCode} {response.StatusCode}");
return(false);
}
// Internal for testing
internal static bool ValidateRelativeLink(Uri uri, IExecutionContext context)
{
List <FilePath> checkPaths = new List <FilePath>();
// Remove the query string and fragment, if any
string normalizedPath = uri.ToString();
if (normalizedPath.Contains("#"))
{
normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("#", StringComparison.Ordinal));
}
if (normalizedPath.Contains("?"))
{
normalizedPath = normalizedPath.Remove(normalizedPath.IndexOf("?", StringComparison.Ordinal));
}
if (normalizedPath == string.Empty)
{
return(true);
}
// Remove the link root if there is one and remove the preceding slash
if (context.Settings.DirectoryPath(Keys.LinkRoot) != null &&
normalizedPath.StartsWith(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath))
{
normalizedPath = normalizedPath.Substring(context.Settings.DirectoryPath(Keys.LinkRoot).FullPath.Length);
}
if (normalizedPath.StartsWith("/"))
{
normalizedPath = normalizedPath.Length > 1 ? normalizedPath.Substring(1) : string.Empty;
}
// Add the base path
if (normalizedPath != string.Empty)
{
checkPaths.Add(new FilePath(normalizedPath));
}
// Add filenames
checkPaths.AddRange(LinkGenerator.DefaultHidePages.Select(x => new FilePath(normalizedPath == string.Empty ? x : $"{normalizedPath}/{x}")));
// Add extensions
checkPaths.AddRange(LinkGenerator.DefaultHideExtensions.SelectMany(x => checkPaths.Select(y => y.AppendExtension(x))).ToArray());
// Check all the candidate paths
FilePath validatedPath = checkPaths.FirstOrDefault(x =>
{
IFile outputFile;
try
{
outputFile = context.FileSystem.GetOutputFile(x);
}
catch (Exception ex)
{
Trace.Warning($"Could not validate path {x.FullPath} for relative link {uri}: {ex.Message}");
return(false);
}
return(outputFile.Exists);
});
if (validatedPath != null)
{
Trace.Verbose($"Validated relative link {uri} at {validatedPath.FullPath}");
return(true);
}
Trace.Warning($"Validation failure for relative link {uri}: could not find output file at any of {string.Join(", ", checkPaths.Select(x => x.FullPath))}");
return(false);
}
/// <inheritdoc />
public IEnumerable <IDocument> Execute(IReadOnlyList <IDocument> inputs, IExecutionContext context)
{
HtmlParser parser = new HtmlParser();
using (IJavaScriptEnginePool enginePool = context.GetJavaScriptEnginePool(x =>
{
if (string.IsNullOrWhiteSpace(_highlightJsFile))
{
x.ExecuteResource("highlight-all.js", typeof(Highlight));
}
else
{
x.ExecuteFile(_highlightJsFile);
}
}))
{
return(inputs.AsParallel().Select(context, input =>
{
try
{
using (Stream stream = input.GetStream())
{
using (IHtmlDocument htmlDocument = parser.Parse(stream))
{
foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector))
{
// Don't highlight anything that potentially is already highlighted
if (element.ClassList.Contains("hljs"))
{
continue;
}
try
{
HighlightElement(enginePool, element);
}
catch (Exception innerEx)
{
if (innerEx.Message.Contains("Unknown language: ") && _warnOnMissingLanguage)
{
Trace.Warning($"Exception while highlighting source code: {innerEx.Message}");
}
else
{
Trace.Information($"Exception while highlighting source code: {innerEx.Message}");
}
}
}
Stream contentStream = context.GetContentStream();
using (StreamWriter writer = contentStream.GetWriter())
{
htmlDocument.ToHtml(writer, HtmlMarkupFormatter.Instance);
writer.Flush();
return context.GetDocument(input, contentStream);
}
}
}
}
catch (Exception ex)
{
Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message);
return input;
}
}).ToList());
}
}
protected override void ParseOptions(ArgumentSyntax syntax)
{
syntax.DefineOption("w|watch", ref _watch, "Watches the input folder for any changes.");
_preview = syntax.DefineOption("p|preview", ref _previewPort, false, "Start the preview web server on the specified port (default is " + _previewPort + ").").IsSpecified;
if (syntax.DefineOption("force-ext", ref _previewForceExtension, "Force the use of extensions in the preview web server (by default, extensionless URLs may be used).").IsSpecified&& !_preview)
{
syntax.ReportError("force-ext can only be specified if the preview server is running.");
}
if (syntax.DefineOption("virtual-dir", ref _previewVirtualDirectory, DirectoryPath.FromString, "Serve files in the preview web server under the specified virtual directory.").IsSpecified&& !_preview)
{
syntax.ReportError("virtual-dir can only be specified if the preview server is running.");
}
if (syntax.DefineOption("preview-root", ref _previewRoot, DirectoryPath.FromString, "The path to the root of the preview server, if not the output folder.").IsSpecified&& !_preview)
{
syntax.ReportError("preview-root can only be specified if the preview server is running.");
}
syntax.DefineOptionList("i|input", ref _configOptions.InputPaths, DirectoryPath.FromString, "The path(s) of input files, can be absolute or relative to the current folder.");
syntax.DefineOption("o|output", ref _configOptions.OutputPath, DirectoryPath.FromString, "The path to output files, can be absolute or relative to the current folder.");
syntax.DefineOption("c|config", ref _configOptions.ConfigFilePath, FilePath.FromString, "Configuration file (by default, config.wyam is used).");
syntax.DefineOption("u|update-packages", ref _configOptions.UpdatePackages, "Check the NuGet server for more recent versions of each package and update them if applicable.");
syntax.DefineOption("use-local-packages", ref _configOptions.UseLocalPackages, "Toggles the use of a local NuGet packages folder.");
syntax.DefineOption("use-global-sources", ref _configOptions.UseGlobalSources, "Toggles the use of the global NuGet sources (default is false).");
syntax.DefineOption("packages-path", ref _configOptions.PackagesPath, DirectoryPath.FromString, "The packages path to use (only if use-local is true).");
syntax.DefineOption("output-script", ref _configOptions.OutputScript, "Outputs the config script after it's been processed for further debugging.");
syntax.DefineOption("verify-config", ref _verifyConfig, false, "Compile the configuration but do not execute.");
syntax.DefineOption("noclean", ref _configOptions.NoClean, "Prevents cleaning of the output path on each execution.");
syntax.DefineOption("nocache", ref _configOptions.NoCache, "Prevents caching information during execution (less memory usage but slower execution).");
_logFilePath = $"wyam-{DateTime.Now:yyyyMMddHHmmssfff}.txt";
if (!syntax.DefineOption("l|log", ref _logFilePath, FilePath.FromString, false, "Log all trace messages to the specified log file (by default, wyam-[datetime].txt).").IsSpecified)
{
_logFilePath = null;
}
// Metadata
// TODO: Remove this dictionary and initial/global options
Dictionary <string, object> settingsDictionary = new Dictionary <string, object>();
IReadOnlyList <string> globalMetadata = null;
if (syntax.DefineOptionList("g|global", ref globalMetadata, "Deprecated, do not use.").IsSpecified)
{
Trace.Warning("-g/--global is deprecated and will be removed in a future version. Please use -s/--setting instead.");
AddSettings(settingsDictionary, globalMetadata);
}
IReadOnlyList <string> initialMetadata = null;
if (syntax.DefineOptionList("initial", ref initialMetadata, "Deprecated, do not use.").IsSpecified)
{
Trace.Warning("--initial is deprecated and will be removed in a future version. Please use -s/--setting instead.");
AddSettings(settingsDictionary, initialMetadata);
}
IReadOnlyList <string> settings = null;
if (syntax.DefineOptionList("s|setting", ref settings, "Specifies a setting as a key=value pair. Use the syntax [x,y] to specify an array value.").IsSpecified)
{
// _configOptions.Settings = MetadataParser.Parse(settings); TODO: Use this when AddSettings() is removed
AddSettings(settingsDictionary, settings);
}
if (settingsDictionary.Count > 0)
{
_configOptions.Settings = settingsDictionary;
}
}
