public ProcessContext(IHostService hs, FileModel fm, IDocumentBuildContext bc = null)
{
_model = fm;
OriginalFileAndType = fm.OriginalFileAndType;
FileAndType = fm.FileAndType;
Uids = new List <UidDefinition>();
UidLinkSources = new Dictionary <string, List <LinkSourceInfo> >();
FileLinkSources = new Dictionary <string, List <LinkSourceInfo> >();
Dependency = new HashSet <string>();
XRefSpecs = new List <XRefSpec>();
ExternalXRefSpecs = new List <XRefSpec>();
if (((IDictionary <string, object>)(fm.Properties)).TryGetValue("PathProperties", out var properties))
{
var pathProperties = properties as Dictionary <string, Dictionary <string, object> >;
PathProperties = pathProperties ?? throw new ArgumentException($"PathProperties is expecting a dictionary however is a {pathProperties.GetType()}");
}
else
{
fm.Properties.PathProperties = PathProperties = new Dictionary <string, Dictionary <string, object> >();
}
Host = hs;
BuildContext = bc;
if (((IDictionary <string, object>)(fm.Properties)).TryGetValue("MarkdigMarkdownService", out var service))
{
MarkdigMarkdownService = (MarkdigMarkdownService)service;
}
}
private static void TransformDocument(string result, string extension, IDocumentBuildContext context, string outputPath, string relativeOutputPath, HashSet <string> missingUids)
{
if (extension.Equals(".html", StringComparison.OrdinalIgnoreCase))
{
try
{
TranformHtml(context, result, relativeOutputPath, outputPath);
}
catch (AggregateException e)
{
e.Handle(s =>
{
var xrefExcetpion = s as CrossReferenceNotResolvedException;
if (xrefExcetpion != null)
{
missingUids.Add(xrefExcetpion.UidRawText);
return(true);
}
else
{
return(false);
}
});
}
}
else
{
File.WriteAllText(outputPath, result, Encoding.UTF8);
}
}
private void UpdateHref(HtmlNode link, string attribute, IDocumentBuildContext context, string sourceFilePath, string destFilePath)
{
var originalHref = link.GetAttributeValue(attribute, null);
var path = UriUtility.GetPath(originalHref);
var anchorFromNode = link.GetAttributeValue("anchor", null);
var segments = anchorFromNode ?? UriUtility.GetQueryStringAndFragment(originalHref);
link.Attributes.Remove("anchor");
if (RelativePath.TryParse(path) == null)
{
if (!string.IsNullOrEmpty(anchorFromNode))
{
link.SetAttributeValue(attribute, originalHref + anchorFromNode);
}
return;
}
var fli = FileLinkInfo.Create(sourceFilePath, destFilePath, path, context);
// fragment and query in original href takes precedence over the one from hrefGenerator
var href = _settings.HrefGenerator?.GenerateHref(fli);
link.SetAttributeValue(
attribute,
href == null ? fli.Href + segments : UriUtility.MergeHref(href, segments));
}
private void TransformXrefInHtml(IDocumentBuildContext context, string sourceFilePath, string destFilePath, HtmlNode node, List <XRefDetails> unresolvedXRefs)
{
var xrefLinkNodes = node.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefNodes = node.SelectNodes("//xref");
var hasResolved = false;
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
var(resolved, warn) = UpdateXref(xref, context, Constants.DefaultLanguage, out var xrefDetails);
if (warn)
{
unresolvedXRefs.Add(xrefDetails);
}
hasResolved = hasResolved || resolved;
}
}
if (hasResolved)
{
// transform again as the resolved content may also contain xref to resolve
TransformXrefInHtml(context, sourceFilePath, destFilePath, node, unresolvedXRefs);
}
}
private void ResolveUid(TocItemViewModel item, FileModel model, IDocumentBuildContext context)
{
if (item.Uid != null)
{
var xref = GetXrefFromUid(item.Uid, model, context);
if (xref != null)
{
item.Href = xref.Href;
if (string.IsNullOrEmpty(item.Name))
{
item.Name = xref.Name;
}
string nameForCSharp;
if (string.IsNullOrEmpty(item.NameForCSharp) && xref.TryGetValue("name.csharp", out nameForCSharp))
{
item.NameForCSharp = nameForCSharp;
}
string nameForVB;
if (string.IsNullOrEmpty(item.NameForVB) && xref.TryGetValue("name.vb", out nameForVB))
{
item.NameForVB = nameForVB;
}
}
}
if (item.HomepageUid != null)
{
item.Homepage = GetXrefFromUid(item.HomepageUid, model, context)?.Href;
}
}
private void TransformDocument(string result, string extension, IDocumentBuildContext context, string destFilePath, ManifestItem manifestItem, out List <XRefDetails> unresolvedXRefs)
{
Task <byte[]> hashTask;
unresolvedXRefs = new List <XRefDetails>();
using (var stream = EnvironmentContext.FileAbstractLayer.Create(destFilePath).WithMd5Hash(out hashTask))
using (var sw = new StreamWriter(stream))
{
if (extension.Equals(".html", StringComparison.OrdinalIgnoreCase))
{
TransformHtml(context, result, manifestItem.SourceRelativePath, destFilePath, sw, out unresolvedXRefs);
}
else
{
sw.Write(result);
}
}
var ofi = new OutputFileInfo
{
RelativePath = destFilePath,
LinkToPath = GetLinkToPath(destFilePath),
Hash = Convert.ToBase64String(hashTask.Result)
};
manifestItem.OutputFiles.Add(extension, ofi);
}
/// <summary>
/// Root toc should always from working folder
/// Parent toc is the first nearest toc
/// </summary>
/// <param name="context">The document build context</param>
/// <param name="item">The manifest item</param>
/// <returns>A class containing root toc path and parent toc path</returns>
private static TocInfo GetTocInfo(IDocumentBuildContext context, ManifestItem item)
{
string key = GetFileKey(item.Key);
RelativePath rootTocPath = null;
RelativePath parentTocPath = null;
var rootToc = context.GetTocFileKeySet(RelativePath.WorkingFolder)?.FirstOrDefault();
var parentToc = context.GetTocFileKeySet(key)?.FirstOrDefault();
if (parentToc == null)
{
// fall back to get the toc file from the same directory
var directory = ((RelativePath)key).GetDirectoryPath();
parentToc = context.GetTocFileKeySet(directory)?.FirstOrDefault();
}
if (rootToc != null)
{
rootTocPath = GetFinalFilePath(rootToc, context);
}
if (parentToc != null)
{
parentTocPath = GetFinalFilePath(parentToc, context);
}
return(new TocInfo(rootTocPath, parentTocPath));
}
private static void TranformHtml(IDocumentBuildContext context, string transformed, string relativeModelPath, StreamWriter outputWriter)
{
// Update HREF and XREF
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
html.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentBuildContext context)
{
if (toc.IsHrefUpdated)
{
return;
}
ResolveUid(toc, model, context);
// Have to register TocMap after uid is resolved
RegisterTocMapToContext(toc, model, context);
toc.Homepage = ResolveHref(toc.Homepage, toc.OriginalHomepage, model, context, nameof(toc.Homepage));
toc.OriginalHomepage = null;
toc.Href = ResolveHref(toc.Href, toc.OriginalHref, model, context, nameof(toc.Href));
toc.OriginalHref = null;
toc.TocHref = ResolveHref(toc.TocHref, toc.OriginalTocHref, model, context, nameof(toc.TocHref));
toc.OriginalTocHref = null;
toc.TopicHref = ResolveHref(toc.TopicHref, toc.OriginalTopicHref, model, context, nameof(toc.TopicHref));
toc.OriginalTopicHref = null;
if (toc.Items != null && toc.Items.Count > 0)
{
foreach (var item in toc.Items)
{
UpdateTocItemHref(item, model, context);
}
}
toc.IsHrefUpdated = true;
}
/// <summary>
/// Root toc should always from working folder
/// Parent toc is the first nearest toc
/// </summary>
/// <param name="context">The document build context</param>
/// <param name="item">The manifest item</param>
/// <returns>A class containing root toc path and parent toc path</returns>
private static TocInfo GetTocInfo(IDocumentBuildContext context, ManifestItem item)
{
string relativePath = item.OriginalFile;
string key = GetFileKey(relativePath);
RelativePath rootTocPath = null;
RelativePath parentTocPath = null;
var rootToc = context.GetTocFileKeySet(RelativePath.WorkingFolder)?.FirstOrDefault();
var parentToc = context.GetTocFileKeySet(key)?.FirstOrDefault();
if (parentToc == null)
{
// fall back to get the toc file from the same directory
var directory = ((RelativePath)key).GetDirectoryPath();
parentToc = context.GetTocFileKeySet(directory)?.FirstOrDefault();
}
if (rootToc != null)
{
rootTocPath = GetFinalFilePath(rootToc, context);
}
if (parentToc != null)
{
parentTocPath = GetFinalFilePath(parentToc, context);
}
return new TocInfo(rootTocPath, parentTocPath);
}
private string ResolveHref(string originalPathToFile, FileModel model, IDocumentBuildContext context)
{
if (!Utility.IsSupportedRelativeHref(originalPathToFile))
{
return(originalPathToFile);
}
var index = originalPathToFile.IndexOf('#');
if (index == 0)
{
throw new DocumentException($"Invalid toc link: {originalPathToFile}.");
}
string href = index == -1
? context.GetFilePath(originalPathToFile)
: context.GetFilePath(originalPathToFile.Remove(index));
if (href == null)
{
Logger.LogWarning($"Unable to find file \"{originalPathToFile}\" referenced by TOC file \"{model.LocalPathFromRepoRoot}\"");
return(originalPathToFile);
}
var relativePath = GetRelativePath(href, model.File);
var path = ((RelativePath)relativePath).UrlEncode().ToString();
if (index >= 0)
{
path += originalPathToFile.Substring(index);
}
return(path);
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var toc = (TocItemViewModel)model.Content;
UpdateTocItemHref(toc, model, context);
RegisterTocToContext(toc, model, context);
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var toc = ConvertFromObject(model.Content);
UpdateTocItemHref(toc, model, context);
RegisterTocToContext(toc, model, context);
model.Content = ConvertToObject(toc);
}
private static void TranformHtml(IDocumentBuildContext context, string transformed, string relativeModelPath, string outputPath)
{
// Update HREF and XREF
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Language);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
// Save with extension changed
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) && !Directory.Exists(subDirectory))
{
Directory.CreateDirectory(subDirectory);
}
html.Save(outputPath, Encoding.UTF8);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
public SystemMetadataGenerator(IDocumentBuildContext context)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
// Order toc files by the output folder depth
_toc = context.GetTocInfo()
.Select(s => new FileInfo(s.TocFileKey, context.GetFilePath(s.TocFileKey)))
.Where(s => s.RelativePath != null)
.OrderBy(s => s.RelativePath.SubdirectoryCount);
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var content = model.Content;
var pc = new ProcessContext(null, model, context);
DocumentSchema schema = model.Properties.Schema;
model.Content = new SchemaProcessor(
new HrefInterpreter(false, true),
new FileInterpreter(false, true)).Process(content, schema, pc);
}
private XRefSpec GetXrefFromUid(string uid, FileModel model, IDocumentBuildContext context)
{
var xref = context.GetXrefSpec(uid);
if (xref == null)
{
Logger.LogWarning($"Unable to find file with uid \"{uid}\" referenced by TOC file \"{model.LocalPathFromRoot}\"");
}
return(xref);
}
private void RegisterTocMap(TocItemViewModel item, string key, IDocumentBuildContext context)
{
var href = item.Href; // Should be original href from working folder starting with ~
if (!PathUtility.IsRelativePath(href))
{
return;
}
context.RegisterToc(key, href);
}
private void TransformHtml(IDocumentBuildContext context, string html, string sourceFilePath, string destFilePath, StreamWriter outputWriter, out List <XRefDetails> unresolvedXRefs)
{
// Update href and xref
HtmlDocument document = new HtmlDocument();
document.LoadHtml(html);
TransformHtmlCore(context, sourceFilePath, destFilePath, document, out unresolvedXRefs);
document.Save(outputWriter);
}
private void UpdateHref(HtmlNode link, string attribute, IDocumentBuildContext context, string sourceFilePath, string destFilePath)
{
var originalHref = link.GetAttributeValue(attribute, null);
var anchor = link.GetAttributeValue("anchor", null);
link.Attributes.Remove("anchor");
var originalPath = UriUtility.GetPath(originalHref);
var path = RelativePath.TryParse(originalPath);
if (path == null)
{
if (!string.IsNullOrEmpty(anchor))
{
link.SetAttributeValue(attribute, originalHref + anchor);
}
return;
}
var fli = new FileLinkInfo
{
FromFileInSource = sourceFilePath,
FromFileInDest = destFilePath,
};
if (path.IsFromWorkingFolder())
{
var targetInSource = path.UrlDecode();
fli.ToFileInSource = targetInSource.RemoveWorkingFolder();
fli.ToFileInDest = RelativePath.GetPathWithoutWorkingFolderChar(context.GetFilePath(targetInSource));
fli.FileLinkInSource = targetInSource - (RelativePath)sourceFilePath;
if (fli.ToFileInDest != null)
{
var resolved = (RelativePath)fli.ToFileInDest - (RelativePath)destFilePath;
fli.FileLinkInDest = resolved;
fli.Href = resolved.UrlEncode();
}
else
{
fli.Href = (targetInSource.RemoveWorkingFolder() - ((RelativePath)sourceFilePath).RemoveWorkingFolder()).UrlEncode();
}
}
else
{
fli.FileLinkInSource = path.UrlDecode();
fli.ToFileInSource = ((RelativePath)sourceFilePath + path).RemoveWorkingFolder();
fli.FileLinkInDest = fli.FileLinkInSource;
fli.Href = originalPath;
}
var href = _settings.HrefGenerator?.GenerateHref(fli) ?? fli.Href;
link.SetAttributeValue(attribute, href + UriUtility.GetQueryString(originalHref) + (anchor ?? UriUtility.GetFragment(originalHref)));
}
private List <CrossReferenceNotResolvedException> TransformHtmlCore(IDocumentBuildContext context, string sourceFilePath, string destFilePath, HtmlAgilityPack.HtmlDocument html)
{
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref")?
.Where(s => s.GetAttributeValue("href", null) != null || s.GetAttributeValue("uid", null) != null).ToList();
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, sourceFilePath, destFilePath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, sourceFilePath, destFilePath);
}
}
return(xrefExceptions);
}
public SystemAttributes(IDocumentBuildContext context, ManifestItem item, string lang)
{
Language = lang;
var tuple = GetTocInfo(context, item);
TocPath = tuple.ParentToc;
RootTocPath = tuple.RootToc;
var file = (RelativePath)item.ModelFile;
TocRelativePath = tuple.ParentToc == null ? null : tuple.ParentToc.MakeRelativeTo(file);
RootTocRelativePath = tuple.RootToc == null ? null : tuple.RootToc.MakeRelativeTo(file);
RelativePathToRoot = (RelativePath.Empty).MakeRelativeTo(file);
PathFromRoot = file.RemoveWorkingFolder();
}
private static List <CrossReferenceNotResolvedException> TransformHtmlCore(IDocumentBuildContext context, string relativeModelPath, HtmlAgilityPack.HtmlDocument html)
{
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
return(xrefExceptions);
}
private XRefSpec GetXrefFromUid(string uid, FileModel model, IDocumentBuildContext context, string includedFrom)
{
var xref = context.GetXrefSpec(uid);
if (xref == null)
{
Logger.LogWarning(
$"Unable to find file with uid \"{uid}\" referenced by TOC file \"{includedFrom ?? model.LocalPathFromRoot}\"",
code: WarningCodes.Build.UidNotFound,
file: includedFrom);
}
return(xref);
}
public SystemAttributes(IDocumentBuildContext context, ManifestItem item, string lang)
{
Language = lang;
var tuple = GetTocInfo(context, item);
TocPath = tuple.ParentToc;
RootTocPath = tuple.RootToc;
var file = (RelativePath)item.ModelFile;
TocRelativePath = tuple.ParentToc == null ? null : tuple.ParentToc.MakeRelativeTo(file);
RootTocRelativePath = tuple.RootToc == null ? null : tuple.RootToc.MakeRelativeTo(file);
RelativePathToRoot = (RelativePath.Empty).MakeRelativeTo(file);
PathFromRoot = file.RemoveWorkingFolder();
}
private void TransformHtml(IDocumentBuildContext context, string html, string sourceFilePath, string destFilePath, StreamWriter outputWriter)
{
// Update href and xref
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
var xrefExceptions = TransformHtmlCore(context, sourceFilePath, destFilePath, document);
document.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
private static void TransformDocument(string result, string extension, IDocumentBuildContext context, string outputPath, string relativeOutputPath, HashSet <string> missingUids, ManifestItem manifestItem)
{
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) &&
!Directory.Exists(subDirectory))
{
Directory.CreateDirectory(subDirectory);
}
Task <byte[]> hashTask;
using (var stream = File.Create(outputPath).WithMd5Hash(out hashTask))
using (var sw = new StreamWriter(stream))
{
if (extension.Equals(".html", StringComparison.OrdinalIgnoreCase))
{
try
{
TranformHtml(context, result, relativeOutputPath, sw);
}
catch (AggregateException e)
{
e.Handle(s =>
{
var xrefExcetpion = s as CrossReferenceNotResolvedException;
if (xrefExcetpion != null)
{
missingUids.Add(xrefExcetpion.UidRawText);
return(true);
}
else
{
return(false);
}
});
}
}
else
{
sw.Write(result);
}
}
manifestItem.OutputFiles.Add(extension, new OutputFileInfo
{
RelativePath = relativeOutputPath,
LinkToPath = null,
Hash = Convert.ToBase64String(hashTask.Result)
});
}
private void UpdateHref(List <ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose($"Updating href...");
manifest.RunAll(m =>
{
using (new LoggerFileScope(m.FileModel.LocalPathFromRepoRoot))
{
Logger.LogVerbose($"Plug-in {m.Processor.Name}: Updating href...");
m.Processor.UpdateHref(m.FileModel, context);
// reset model after updating href
m.Item.Model = m.FileModel.ModelWithCache;
}
});
}
public SystemMetadataGenerator(IDocumentBuildContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_context = context;
// Order toc files by the output folder depth
_toc = context.GetTocInfo()
.Select(s => new FileInfo(s.TocFileKey, (RelativePath)context.GetFilePath(s.TocFileKey)))
.Where(s => s.File != null)
.OrderBy(s => s.File.SubdirectoryCount);
}
private void TransformHtmlCore(IDocumentBuildContext context, string sourceFilePath, string destFilePath, HtmlDocument html, out List <XRefDetails> unresolvedXRefs)
{
unresolvedXRefs = new List <XRefDetails>();
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefNodes = html.DocumentNode.SelectNodes("//xref");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
var resolved = UpdateXref(xref, context, Constants.DefaultLanguage, out var xrefDetails);
if (!resolved)
{
unresolvedXRefs.Add(xrefDetails);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, sourceFilePath, destFilePath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, sourceFilePath, destFilePath);
}
}
}
private void FeedOptions(List <ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose($"Feeding options from template...");
manifest.RunAll(m =>
{
if (m.TemplateBundle == null)
{
return;
}
using (new LoggerFileScope(m.FileModel.LocalPathFromRepoRoot))
{
Logger.LogVerbose($"Feed options from template for {m.Item.DocumentType}...");
m.Options = m.TemplateBundle.GetOptions(m.Item, context);
}
});
}
private static Manifest GenerateManifest(IDocumentBuildContext context, List <TemplateManifestItem> items)
{
var toc = context.GetTocInfo();
var homepages = toc
.Where(s => !string.IsNullOrEmpty(s.Homepage))
.Select(s => new HomepageInfo
{
Homepage = RelativePath.GetPathWithoutWorkingFolderChar(s.Homepage),
TocPath = RelativePath.GetPathWithoutWorkingFolderChar(context.GetFilePath(s.TocFileKey))
}).ToList();
return(new Manifest
{
Homepages = homepages,
Files = items,
});
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var toc = (TocViewModel)model.Content;
var key = model.Key;
// Add current folder to the toc mapping, e.g. `a/` maps to `a/toc`
var directory = ((RelativePath)key).GetPathFromWorkingFolder().GetDirectoryPath();
context.RegisterToc(key, directory);
if (toc.Count > 0)
{
foreach (var item in toc)
{
UpdateTocItemHref(item, model, context);
}
}
}
private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentBuildContext context)
{
ResolveUid(toc, model, context);
// Have to register TocMap after uid is resolved
RegisterTocMap(toc, model.Key, context);
toc.Homepage = ResolveHref(toc.Homepage, model, context);
toc.Href = ResolveHref(toc.Href, model, context);
toc.TocHref = ResolveHref(toc.TocHref, model, context);
if (toc.Items != null && toc.Items.Count > 0)
{
foreach (var item in toc.Items)
{
UpdateTocItemHref(item, model, context);
}
}
}
private IEnumerable<TransformModelOptions> GetOptionsForEachTemplate(InternalManifestItem item, IDocumentBuildContext context)
{
if (item == null)
{
yield break;
}
foreach (var template in Templates)
{
if (template.ContainsGetOptions)
{
var options = template.GetOptions(item.Model.Content);
if (options != null)
{
yield return options;
}
}
}
}
private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentBuildContext context)
{
ResolveUid(toc, model, context);
RegisterTocMap(toc, model.Key, context);
if (!string.IsNullOrEmpty(toc.Homepage))
{
toc.Href = toc.Homepage;
}
toc.Href = GetUpdatedHref(toc.Href, model, context);
toc.OriginalHref = GetUpdatedHref(toc.OriginalHref, model, context);
if (toc.Items != null && toc.Items.Count > 0)
{
foreach (var item in toc.Items)
{
UpdateTocItemHref(item, model, context);
}
}
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var toc = (TocItemViewModel)model.Content;
var key = model.Key;
// Add current folder to the toc mapping, e.g. `a/` maps to `a/toc`
var directory = ((TypeForwardedToRelativePath)key).GetPathFromWorkingFolder().GetDirectoryPath();
context.RegisterToc(key, directory);
UpdateTocItemHref(toc, model, context);
var tocInfo = new TocInfo(key);
if (toc.Homepage != null)
{
if (TypeForwardedToPathUtility.IsRelativePath(toc.Homepage))
{
var pathToRoot = ((TypeForwardedToRelativePath)model.File + (TypeForwardedToRelativePath)toc.Homepage).GetPathFromWorkingFolder();
tocInfo.Homepage = pathToRoot;
}
}
context.RegisterTocInfo(tocInfo);
}
private void UpdateHref(List<ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose("Updating href...");
manifest.RunAll(m =>
{
using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Plug-in {m.Processor.Name}: Updating href...");
m.Processor.UpdateHref(m.FileModel, context);
// reset model after updating href
m.Item.Model = m.FileModel.ModelWithCache;
}
});
}
private static void TransformXrefLink(HtmlAgilityPack.HtmlNode node, IDocumentBuildContext context)
{
var convertedNode = XRefDetails.ConvertXrefLinkNodeToXrefNode(node);
node.ParentNode.ReplaceChild(convertedNode, node);
}
private static void UpdateXref(HtmlAgilityPack.HtmlNode node, IDocumentBuildContext context, string language)
{
var xref = XRefDetails.From(node);
// Resolve external xref map first, and then internal xref map.
// Internal one overrides external one
var xrefSpec = context.GetXrefSpec(xref.Uid);
xref.ApplyXrefSpec(xrefSpec);
var convertedNode = xref.ConvertToHtmlNode(language);
node.ParentNode.ReplaceChild(convertedNode, node);
if (xrefSpec == null)
{
if (xref.ThrowIfNotResolved)
{
throw new CrossReferenceNotResolvedException(xref.Uid, xref.RawSource, null);
}
}
}
private void RegisterTocMap(TocItemViewModel item, string key, IDocumentBuildContext context)
{
var href = item.Href; // Should be original href from working folder starting with ~
if (!PathUtility.IsRelativePath(href)) return;
context.RegisterToc(key, href);
}
private void FeedXRefMap(List<ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose("Feeding xref map...");
manifest.RunAll(m =>
{
if (m.TemplateBundle == null)
{
return;
}
using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Feed xref map from template for {m.Item.DocumentType}...");
// TODO: use m.Options.Bookmarks directly after all templates report bookmarks
var bookmarks = m.Options.Bookmarks ?? m.FileModel.Bookmarks;
foreach (var pair in bookmarks)
{
context.RegisterInternalXrefSpecBookmark(pair.Key, pair.Value);
}
}
});
}
private static void TransformDocument(string result, string extension, IDocumentBuildContext context, string outputPath, string relativeOutputPath, HashSet<string> missingUids, ManifestItem manifestItem)
{
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) &&
!Directory.Exists(subDirectory))
{
Directory.CreateDirectory(subDirectory);
}
Task<byte[]> hashTask;
using (var stream = File.Create(outputPath).WithMd5Hash(out hashTask))
using (var sw = new StreamWriter(stream))
{
if (extension.Equals(".html", StringComparison.OrdinalIgnoreCase))
{
try
{
TransformHtml(context, result, relativeOutputPath, sw);
}
catch (AggregateException e)
{
e.Handle(s =>
{
var xrefExcetpion = s as CrossReferenceNotResolvedException;
if (xrefExcetpion != null)
{
missingUids.Add(xrefExcetpion.UidRawText);
return true;
}
else
{
return false;
}
});
}
}
else
{
sw.Write(result);
}
}
manifestItem.OutputFiles.Add(extension, new OutputFileInfo
{
RelativePath = relativeOutputPath,
LinkToPath = null,
Hash = Convert.ToBase64String(hashTask.Result)
});
}
private void ApplySystemMetadata(List<ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose("Applying system metadata to manifest...");
// Add system attributes
var systemMetadataGenerator = new SystemMetadataGenerator(context);
manifest.RunAll(m =>
{
using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic("Generating system metadata...");
// TODO: use weak type for system attributes from the beginning
var systemAttrs = systemMetadataGenerator.Generate(m.Item);
var metadata = (IDictionary<string, object>)ConvertToObjectHelper.ConvertStrongTypeToObject(systemAttrs);
// Change file model to weak type
var model = m.Item.Model.Content;
var modelAsObject = ConvertToObjectHelper.ConvertStrongTypeToObject(model) as IDictionary<string, object>;
if (modelAsObject != null)
{
foreach (var token in modelAsObject)
{
// Overwrites the existing system metadata if the same key is defined in document model
metadata[token.Key] = token.Value;
}
}
else
{
Logger.LogWarning("Input model is not an Object model, it will be wrapped into an Object model. Please use --exportRawModel to view the wrapped model");
metadata["model"] = model;
}
// Append system metadata to model
m.Item.Model.Content = metadata;
}
});
}
private void RegisterTocMap(TocItemViewModel item, string key, IDocumentBuildContext context)
{
// If tocHref is set, href is originally RelativeFolder type, and href is set to the homepage of TocHref,
// So in this case, TocHref should be used to in TocMap
// TODO: what if user wants to set TocHref?
var tocHref = item.TocHref;
var tocHrefType = Utility.GetHrefType(tocHref);
if (tocHrefType == HrefType.MarkdownTocFile || tocHrefType == HrefType.YamlTocFile)
{
context.RegisterToc(key, tocHref);
}
else
{
var href = item.Href; // Should be original href from working folder starting with ~
if (Utility.IsSupportedRelativeHref(href))
{
context.RegisterToc(key, href);
}
}
}
private string ResolveHref(string pathToFile, string originalPathToFile, FileModel model, IDocumentBuildContext context, string propertyName)
{
if (!Utility.IsSupportedRelativeHref(pathToFile))
{
return pathToFile;
}
var index = pathToFile.IndexOfAny(QueryStringOrAnchor);
if (index == 0)
{
throw new DocumentException($"Invalid toc link for {propertyName}: {originalPathToFile}.");
}
string href = index == -1
? context.GetFilePath(pathToFile)
: context.GetFilePath(pathToFile.Remove(index));
if (href == null)
{
Logger.LogInfo($"Unable to find file \"{originalPathToFile}\" for {propertyName} referenced by TOC file \"{model.LocalPathFromRoot}\"");
return originalPathToFile;
}
var relativePath = GetRelativePath(href, model.File);
var path = ((TypeForwardedToRelativePath)relativePath).UrlEncode().ToString();
if (index >= 0)
{
path += pathToFile.Substring(index);
}
return path;
}
internal TransformModelOptions GetOptions(InternalManifestItem item, IDocumentBuildContext context)
{
return MergeOptions(GetOptionsForEachTemplate(item, context));
}
private XRefSpec GetXrefFromUid(string uid, FileModel model, IDocumentBuildContext context)
{
var xref = context.GetXrefSpec(uid);
if (xref == null)
{
throw new DocumentException($"Unable to find file with uid \"{uid}\" referenced by TOC file \"{model.LocalPathFromRepoRoot}\"");
}
return xref;
}
private static RelativePath GetFinalFilePath(string key, IDocumentBuildContext context)
{
var fileKey = GetFileKey(key);
return ((RelativePath)context.GetFilePath(fileKey)).RemoveWorkingFolder();
}
private void ResolveUid(TocItemViewModel item, FileModel model, IDocumentBuildContext context)
{
if (item.Uid != null)
{
var xref = GetXrefFromUid(item.Uid, model, context);
item.Href = xref.Href;
if (string.IsNullOrEmpty(item.Name))
{
item.Name = xref.Name;
}
string nameForCSharp;
if (string.IsNullOrEmpty(item.NameForCSharp) && xref.TryGetValue("name.csharp", out nameForCSharp))
{
item.NameForCSharp = nameForCSharp;
}
string nameForVB;
if (string.IsNullOrEmpty(item.NameForVB) && xref.TryGetValue("name.vb", out nameForVB))
{
item.NameForVB = nameForVB;
}
}
if (item.HomepageUid != null)
{
item.Homepage = GetXrefFromUid(item.HomepageUid, model, context).Href;
}
}
private static List<CrossReferenceNotResolvedException> TransformHtmlCore(IDocumentBuildContext context, string relativeModelPath, HtmlAgilityPack.HtmlDocument html)
{
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List<CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
return xrefExceptions;
}
public void UpdateHref(FileModel model, IDocumentBuildContext context)
{
}
private static void TransformHtml(IDocumentBuildContext context, string html, string relativeModelPath, StreamWriter outputWriter)
{
// Update href and xref
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
var xrefExceptions = TransformHtmlCore(context, relativeModelPath, document);
document.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
private void UpdateTocItemHref(TocItemViewModel toc, FileModel model, IDocumentBuildContext context)
{
if (toc.IsHrefUpdated) return;
ResolveUid(toc, model, context);
// Have to register TocMap after uid is resolved
RegisterTocMap(toc, model.Key, context);
toc.Homepage = ResolveHref(toc.Homepage, toc.OriginalHomepage, model, context, nameof(toc.Homepage));
toc.Href = ResolveHref(toc.Href, toc.OriginalHref, model, context, nameof(toc.Href));
toc.TocHref = ResolveHref(toc.TocHref, toc.OriginalTocHref, model, context, nameof(toc.TocHref));
toc.TopicHref = ResolveHref(toc.TopicHref, toc.OriginalTopicHref, model, context, nameof(toc.TopicHref));
if (toc.Items != null && toc.Items.Count > 0)
{
foreach (var item in toc.Items)
{
UpdateTocItemHref(item, model, context);
}
}
toc.IsHrefUpdated = true;
}
private void FeedOptions(List<ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose("Feeding options from template...");
manifest.RunAll(m =>
{
if (m.TemplateBundle == null)
{
return;
}
using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Feed options from template for {m.Item.DocumentType}...");
m.Options = m.TemplateBundle.GetOptions(m.Item, context);
}
});
}
private static void UpdateHref(HtmlAgilityPack.HtmlNode link, string attribute, IDocumentBuildContext context, string relativePath)
{
var originalHref = link.GetAttributeValue(attribute, null);
var anchor = link.GetAttributeValue("anchor", null);
link.Attributes.Remove("anchor");
string href;
var path = RelativePath.TryParse(originalHref);
if (path?.IsFromWorkingFolder() == true)
{
var targetPath = (RelativePath)context.GetFilePath(path.UrlDecode());
if (targetPath != null)
{
href = (targetPath.RemoveWorkingFolder() - (RelativePath)relativePath).UrlEncode();
}
else
{
Logger.LogInfo($"File {path} is not found in {relativePath}.");
// TODO: what to do if file path not exists?
// CURRENT: fallback to the original one
href = (path.UrlDecode().RemoveWorkingFolder() - (RelativePath)relativePath).UrlEncode();
}
link.SetAttributeValue(attribute, href + anchor);
}
}
private IDictionary<string, object> FeedGlobalVariables(IDictionary<string, string> initialGlobalVariables, List<ManifestItemWithContext> manifest, IDocumentBuildContext context)
{
Logger.LogVerbose("Feeding global variables from template...");
// E.g. we can set TOC model to be globally shared by every data model
// Make sure it is single thread
IDictionary<string, object> metadata = initialGlobalVariables == null ?
new Dictionary<string, object>() :
initialGlobalVariables.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
var sharedObjects = new Dictionary<string, object>();
manifest.RunAll(m =>
{
if (m.TemplateBundle == null)
{
return;
}
using (new LoggerFileScope(m.FileModel.LocalPathFromRoot))
{
Logger.LogDiagnostic($"Load shared model from template for {m.Item.DocumentType}...");
if (m.Options.IsShared)
{
sharedObjects[m.Item.Key] = m.Item.Model.Content;
}
}
});
metadata["_shared"] = sharedObjects;
return metadata;
}
private string GetUpdatedHref(string originalPathToFile, FileModel model, IDocumentBuildContext context)
{
if (!PathUtility.IsRelativePath(originalPathToFile)) return originalPathToFile;
string href = context.GetFilePath(originalPathToFile);
if (href == null)
{
throw new DocumentException($"Unalbe to find file \"{originalPathToFile}\" referenced by TOC file \"{model.LocalPathFromRepoRoot}\"");
}
var relativePath = GetRelativePath(href, model.File);
return relativePath;
}
