private void ReportPreBuildDependency(List <FileModel> models, IHostService host, int parallelism, HashSet <string> includedTocs)
{
var nearest = new ConcurrentDictionary <string, RelativeInfo>(FilePathComparer.OSPlatformSensitiveStringComparer);
models.RunAll(model =>
{
var item = (TocItemViewModel)model.Content;
UpdateNearestToc(host, item, model, nearest);
},
parallelism);
// handle not-in-toc items
UpdateNearestTocForNotInTocItem(models, host, nearest, parallelism);
foreach (var item in nearest)
{
if (includedTocs.Contains(item.Key))
{
host.ReportDependencyTo(item.Value.TocInfo.Model, item.Key, DependencyTypeName.Include);
}
else
{
host.ReportDependencyFrom(item.Value.TocInfo.Model, item.Key, DependencyTypeName.Metadata);
}
}
}
protected override void BuildArticle(IHostService host, FileModel model)
{
var content = model.Content;
var context = new ProcessContext(host, model);
DocumentSchema schema = model.Properties.Schema;
content = _schemaProcessor.Process(content, schema, context);
model.LinkToUids = model.LinkToUids.Union(context.UidLinkSources.Keys);
model.LinkToFiles = model.LinkToFiles.Union(context.FileLinkSources.Keys);
model.FileLinkSources = model.FileLinkSources.Merge(context.FileLinkSources);
model.UidLinkSources = model.UidLinkSources.Merge(context.UidLinkSources);
model.Uids = model.Uids.AddRange(context.Uids);
model.Properties.XRefSpecs = context.XRefSpecs;
model.Properties.ExternalXRefSpecs = context.ExternalXRefSpecs;
foreach (var d in context.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
if (content is IDictionary <string, object> eo)
{
if (eo.TryGetValue(DocumentTypeKey, out object documentType) && documentType is string dt)
{
model.DocumentType = dt;
}
}
}
protected override void BuildArticle(IHostService host, FileModel model)
{
var content = model.Content;
var context = new ProcessContext(host, model);
context.Properties.Uids = new List <UidDefinition>();
context.Properties.UidLinkSources = new Dictionary <string, List <LinkSourceInfo> >();
context.Properties.FileLinkSources = new Dictionary <string, List <LinkSourceInfo> >();
context.Properties.Dependency = new HashSet <string>();
DocumentSchema schema = model.Properties.Schema;
content = _schemaProcessor.Process(content, schema, context);
model.LinkToUids = model.LinkToUids.Union(((Dictionary <string, List <LinkSourceInfo> >)context.Properties.UidLinkSources).Keys);
model.LinkToFiles = model.LinkToFiles.Union(((Dictionary <string, List <LinkSourceInfo> >)context.Properties.FileLinkSources).Keys);
model.FileLinkSources = model.FileLinkSources.Merge((Dictionary <string, List <LinkSourceInfo> >)context.Properties.FileLinkSources);
model.UidLinkSources = model.UidLinkSources.Merge((Dictionary <string, List <LinkSourceInfo> >)context.Properties.UidLinkSources);
foreach (var d in context.Properties.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
if (content is IDictionary <string, object> eo)
{
if (eo.TryGetValue(DocumentTypeKey, out object documentType) && documentType is string dt)
{
model.DocumentType = dt;
}
}
}
/// <summary>
/// TODO: move to Base and share with other DocumentBuilder
/// </summary>
/// <param name="host"></param>
/// <param name="model"></param>
protected override void BuildArticle(IHostService host, FileModel model)
{
var pageViewModel = (PageViewModel)model.Content;
var context = new HandleModelAttributesContext
{
EnableContentPlaceholder = false,
Host = host,
FileAndType = model.OriginalFileAndType,
SkipMarkup = pageViewModel?.ShouldSkipMarkup ?? false,
};
_handler.Handle(pageViewModel, context);
model.LinkToUids = model.LinkToUids.Union(context.LinkToUids);
model.LinkToFiles = model.LinkToFiles.Union(context.LinkToFiles);
model.FileLinkSources = model.FileLinkSources.ToDictionary(v => v.Key, v => v.Value.ToList())
.Merge(context.FileLinkSources.Select(i => new KeyValuePair <string, IEnumerable <LinkSourceInfo> >(i.Key, i.Value)))
.ToImmutableDictionary(v => v.Key, v => v.Value.ToImmutableList());
model.UidLinkSources = model.UidLinkSources.ToDictionary(v => v.Key, v => v.Value.ToList())
.Merge(context.UidLinkSources.Select(i => new KeyValuePair <string, IEnumerable <LinkSourceInfo> >(i.Key, i.Value)))
.ToImmutableDictionary(v => v.Key, v => v.Value.ToImmutableList());
foreach (var r in pageViewModel.References)
{
if (r.IsExternal == false)
{
host.ReportDependencyTo(model, r.Uid, DependencyItemSourceType.Uid, DependencyTypeName.Reference);
}
}
}
protected virtual void BuildArticleCore(IHostService host, FileModel model, IModelAttributeHandler handlers = null, HandleModelAttributesContext handlerContext = null, bool shouldSkipMarkup = false)
{
if (handlers == null)
{
handlers = _defaultHandler;
}
if (handlerContext == null)
{
handlerContext = new HandleModelAttributesContext
{
EnableContentPlaceholder = false,
Host = host,
FileAndType = model.OriginalFileAndType,
SkipMarkup = shouldSkipMarkup,
};
}
handlers.Handle(model.Content, handlerContext);
model.LinkToUids = model.LinkToUids.Union(handlerContext.LinkToUids);
model.LinkToFiles = model.LinkToFiles.Union(handlerContext.LinkToFiles);
model.FileLinkSources = model.FileLinkSources.ToDictionary(v => v.Key, v => v.Value.ToList())
.Merge(handlerContext.FileLinkSources.Select(i => new KeyValuePair <string, IEnumerable <LinkSourceInfo> >(i.Key, i.Value)))
.ToImmutableDictionary(v => v.Key, v => v.Value.ToImmutableList());
model.UidLinkSources = model.UidLinkSources.ToDictionary(v => v.Key, v => v.Value.ToList())
.Merge(handlerContext.UidLinkSources.Select(i => new KeyValuePair <string, IEnumerable <LinkSourceInfo> >(i.Key, i.Value)))
.ToImmutableDictionary(v => v.Key, v => v.Value.ToImmutableList());
foreach (var d in handlerContext.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
}
public override void Build(FileModel model, IHostService host)
{
if (model.MarkdownFragmentsModel == null)
{
return;
}
host.ReportDependencyTo(model, model.MarkdownFragmentsModel.Key, DependencyTypeName.Include);
if (model.MarkdownFragmentsModel.Content == null)
{
return;
}
CheckMarkdownService(host);
if (!(model.MarkdownFragmentsModel.Content is string))
{
var message = "Unable to parse markdown fragments. Expect string content.";
Logger.LogError(message);
throw new DocfxException(message);
}
if (model.MarkdownFragmentsModel.Properties.MarkdigMarkdownService == null || !(model.MarkdownFragmentsModel.Properties.MarkdigMarkdownService is MarkdigMarkdownService))
{
var message = "Unable to find markdig markdown service in file model.";
Logger.LogError(message);
throw new DocfxException(message);
}
if (!(model.Properties.Schema is DocumentSchema))
{
var message = "Unable to find schema in file model.";
Logger.LogError(message);
throw new DocfxException(message);
}
using (new LoggerFileScope(model.MarkdownFragmentsModel.LocalPathFromRoot))
{
try
{
BuildCore(model, host);
}
catch (MarkdownFragmentsException ex)
{
Logger.LogWarning(
$"Unable to parse markdown fragments: {ex.Message}",
line: ex.Position == -1 ? null : (ex.Position + 1).ToString(),
code: WarningCodes.Overwrite.InvalidMarkdownFragments);
return;
}
catch (DocumentException de)
{
Logger.LogError(de.Message);
throw;
}
}
}
public override void Build(FileModel model, IHostService host)
{
if (model.Type != DocumentType.Article)
{
return;
}
var pageViewModel = (PageViewModel)model.Content;
foreach (var uid in GetUidsToFill(pageViewModel))
{
host.ReportDependencyTo(model, uid, DependencyItemSourceType.Uid, DependencyTypeName.Children);
}
}
protected override void BuildArticle(IHostService host, FileModel model)
{
var pageViewModel = (PageViewModel)model.Content;
BuildArticleCore(host, model, shouldSkipMarkup: pageViewModel?.ShouldSkipMarkup ?? false);
foreach (var r in pageViewModel.References)
{
if (r.IsExternal == false)
{
host.ReportDependencyTo(model, r.Uid, DependencyItemSourceType.Uid, DependencyTypeName.Reference);
}
}
}
private object BuildOverwriteWithSchema(FileModel owModel, OverwriteDocumentModel overwrite, IHostService host, BaseSchema schema)
{
dynamic overwriteObject = ConvertToObjectHelper.ConvertToDynamic(overwrite.Metadata);
overwriteObject.uid = overwrite.Uid;
var overwriteModel = new FileModel(owModel.FileAndType, overwriteObject, owModel.OriginalFileAndType);
var context = new ProcessContext(host, overwriteModel)
{
ContentAnchorParser = new ContentAnchorParser(overwrite.Conceptual)
};
var transformed = _overwriteProcessor.Process(overwriteObject, schema, context) as IDictionary <string, object>;
if (!context.ContentAnchorParser.ContainsAnchor)
{
transformed["conceptual"] = context.ContentAnchorParser.Content;
}
// add SouceDetail back to transformed, in week type
transformed[Constants.PropertyName.Documentation] = new Dictionary <string, object>
{
["remote"] = overwrite.Documentation.Remote == null ? null : new Dictionary <string, object>
{
["path"] = overwrite.Documentation.Remote.RelativePath,
["branch"] = overwrite.Documentation.Remote.RemoteBranch,
["repo"] = overwrite.Documentation.Remote.RemoteRepositoryUrl,
}
["path"] = overwrite.Documentation?.Path,
["startLine"] = overwrite.Documentation?.StartLine ?? 0,
["endLine"] = overwrite.Documentation?.EndLine ?? 0,
};
owModel.LinkToUids = owModel.LinkToUids.Union((context.UidLinkSources).Keys);
owModel.LinkToFiles = owModel.LinkToFiles.Union((context.FileLinkSources).Keys);
owModel.FileLinkSources = owModel.FileLinkSources.Merge(context.FileLinkSources);
owModel.UidLinkSources = owModel.UidLinkSources.Merge(context.UidLinkSources);
owModel.Uids = owModel.Uids.AddRange(context.Uids);
owModel.Properties.XRefSpecs = context.XRefSpecs;
owModel.Properties.ExternalXRefSpecs = context.ExternalXRefSpecs;
foreach (var d in context.Dependency)
{
host.ReportDependencyTo(owModel, d, DependencyTypeName.Include);
}
return(transformed);
}
protected virtual void BuildOverwrite(IHostService host, FileModel model)
{
var overwrites = MarkdownReader.ReadMarkdownAsOverwrite(host, model.FileAndType).ToList();
model.Content = overwrites;
model.LinkToFiles = overwrites.SelectMany(o => o.LinkToFiles).ToImmutableHashSet();
model.LinkToUids = overwrites.SelectMany(o => o.LinkToUids).ToImmutableHashSet();
model.FileLinkSources = overwrites.SelectMany(o => o.FileLinkSources).GroupBy(i => i.Key, i => i.Value).ToImmutableDictionary(i => i.Key, i => i.SelectMany(l => l).ToImmutableList());
model.UidLinkSources = overwrites.SelectMany(o => o.UidLinkSources).GroupBy(i => i.Key, i => i.Value).ToImmutableDictionary(i => i.Key, i => i.SelectMany(l => l).ToImmutableList());
model.Uids = (from item in overwrites
select new UidDefinition(
item.Uid,
model.LocalPathFromRoot,
item.Documentation.StartLine + 1)).ToImmutableArray();
foreach (var d in overwrites.SelectMany(s => s.Dependency))
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
}
public override void Build(FileModel model, IHostService host)
{
if (model.Type != DocumentType.Article)
{
return;
}
var content = (Dictionary<string, object>)model.Content;
var markdown = (string)content[ConceptualKey];
var result = host.Markup(markdown, model.FileAndType);
var htmlInfo = SeparateHtml(result.Html);
model.Properties.IsUserDefinedTitle = false;
content[Constants.PropertyName.Title] = htmlInfo.Title;
content["rawTitle"] = htmlInfo.RawTitle;
content[ConceptualKey] = htmlInfo.Content;
if (result.YamlHeader != null && result.YamlHeader.Count > 0)
{
foreach (var item in result.YamlHeader)
{
if (item.Key == Constants.PropertyName.Uid)
{
var uid = item.Value as string;
if (!string.IsNullOrWhiteSpace(uid))
{
model.Uids = new[] { new UidDefinition(uid, model.LocalPathFromRoot) }.ToImmutableArray();
content[Constants.PropertyName.Uid] = item.Value;
}
}
else
{
content[item.Key] = item.Value;
if (item.Key == DocumentTypeKey)
{
model.DocumentType = item.Value as string;
}
if (item.Key == Constants.PropertyName.Title)
{
model.Properties.IsUserDefinedTitle = true;
}
}
}
}
model.LinkToFiles = result.LinkToFiles.ToImmutableHashSet();
model.LinkToUids = result.LinkToUids;
model.FileLinkSources = result.FileLinkSources;
model.UidLinkSources = result.UidLinkSources;
model.Properties.XrefSpec = null;
if (model.Uids.Length > 0)
{
model.Properties.XrefSpec = new XRefSpec
{
Uid = model.Uids[0].Name,
Name = TitleThumbnail(content[Constants.PropertyName.Title] as string ?? model.Uids[0].Name, TitleThumbnailMaxLength),
Href = ((RelativePath)model.File).GetPathFromWorkingFolder()
};
}
foreach (var d in result.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
}
public override void Build(FileModel model, IHostService host)
{
if (model.Type != DocumentType.Article)
{
return;
}
var content = (Dictionary <string, object>)model.Content;
var markdown = (string)content[ConceptualKey];
var result = host.Markup(markdown, model.OriginalFileAndType, false, true);
var htmlInfo = HtmlDocumentUtility.SeparateHtml(result.Html);
model.Properties.IsUserDefinedTitle = false;
content[Constants.PropertyName.Title] = htmlInfo.Title;
content["rawTitle"] = htmlInfo.RawTitle;
if (!string.IsNullOrEmpty(htmlInfo.RawTitle))
{
model.ManifestProperties.rawTitle = htmlInfo.RawTitle;
}
content[ConceptualKey] = htmlInfo.Content;
if (result.YamlHeader?.Count > 0)
{
foreach (var item in result.YamlHeader)
{
HandleKeyValuePair(item.Key, item.Value);
}
}
model.LinkToFiles = result.LinkToFiles.ToImmutableHashSet();
model.LinkToUids = result.LinkToUids;
model.FileLinkSources = result.FileLinkSources;
model.UidLinkSources = result.UidLinkSources;
model.Properties.XrefSpec = null;
if (model.Uids.Length > 0)
{
var title = content[Constants.PropertyName.Title] as string;
model.Properties.XrefSpec = new XRefSpec
{
Uid = model.Uids[0].Name,
Name = string.IsNullOrEmpty(title) ? model.Uids[0].Name : title,
Href = ((RelativePath)model.File).GetPathFromWorkingFolder()
};
}
foreach (var d in result.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
void HandleKeyValuePair(string key, object value)
{
switch (key)
{
case Constants.PropertyName.Uid:
var uid = value as string;
if (!string.IsNullOrWhiteSpace(uid))
{
model.Uids = new[] { new UidDefinition(uid, model.LocalPathFromRoot) }.ToImmutableArray();
content[Constants.PropertyName.Uid] = value;
}
break;
case DocumentTypeKey:
content[key] = value;
model.DocumentType = value as string;
break;
case Constants.PropertyName.Title:
if (value is string str && !string.IsNullOrEmpty(str))
{
content[key] = str;
model.Properties.IsUserDefinedTitle = true;
}
break;
case Constants.PropertyName.OutputFileName:
content[key] = value;
var outputFileName = value as string;
if (!string.IsNullOrWhiteSpace(outputFileName))
{
string fn = null;
try
{
fn = Path.GetFileName(outputFileName);
}
catch (ArgumentException) { }
if (fn == outputFileName)
{
model.File = (RelativePath)model.File + (RelativePath)outputFileName;
}
else
{
Logger.LogWarning($"Invalid output file name in yaml header: {outputFileName}, skip rename output file.");
}
}
break;
default:
content[key] = value;
break;
}
}
}
public override void Build(FileModel model, IHostService host)
{
if (model.Type != DocumentType.Article)
{
return;
}
var content = (Dictionary <string, object>)model.Content;
var markdown = (string)content[ConceptualKey];
var result = host.Markup(markdown, model.FileAndType);
var htmlInfo = SeparateHtml(result.Html);
model.Properties.IsUserDefinedTitle = false;
content[Constants.PropertyName.Title] = htmlInfo.Title;
content["rawTitle"] = htmlInfo.RawTitle;
content[ConceptualKey] = htmlInfo.Content;
if (result.YamlHeader != null && result.YamlHeader.Count > 0)
{
foreach (var item in result.YamlHeader)
{
if (item.Key == Constants.PropertyName.Uid)
{
var uid = item.Value as string;
if (!string.IsNullOrWhiteSpace(uid))
{
model.Uids = new[] { new UidDefinition(uid, model.LocalPathFromRoot) }.ToImmutableArray();
content[Constants.PropertyName.Uid] = item.Value;
}
}
else
{
content[item.Key] = item.Value;
if (item.Key == DocumentTypeKey)
{
model.DocumentType = item.Value as string;
}
if (item.Key == Constants.PropertyName.Title)
{
model.Properties.IsUserDefinedTitle = true;
}
}
}
}
model.LinkToFiles = result.LinkToFiles.ToImmutableHashSet();
model.LinkToUids = result.LinkToUids;
model.FileLinkSources = result.FileLinkSources;
model.UidLinkSources = result.UidLinkSources;
model.Properties.XrefSpec = null;
if (model.Uids.Length > 0)
{
model.Properties.XrefSpec = new XRefSpec
{
Uid = model.Uids[0].Name,
Name = TitleThumbnail(content[Constants.PropertyName.Title] as string ?? model.Uids[0].Name, TitleThumbnailMaxLength),
Href = ((RelativePath)model.File).GetPathFromWorkingFolder()
};
}
foreach (var d in result.Dependency)
{
host.ReportDependencyTo(model, d, DependencyTypeName.Include);
}
}
public object BuildOverwriteWithSchema(FileModel owModel, OverwriteDocumentModel overwrite, BaseSchema schema)
{
if (overwrite == null || owModel == null)
{
return(null);
}
if (schema == null)
{
throw new ArgumentNullException(nameof(schema));
}
dynamic overwriteObject = ConvertToObjectHelper.ConvertToDynamic(overwrite.Metadata);
overwriteObject.uid = overwrite.Uid;
var overwriteModel = new FileModel(owModel.FileAndType, overwriteObject, owModel.OriginalFileAndType);
var context = (((IDictionary <string, object>)(owModel.Properties)).TryGetValue("MarkdigMarkdownService", out var service))
? new ProcessContext(_host, overwriteModel, null, (MarkdigMarkdownService)service)
: new ProcessContext(_host, overwriteModel);
if (_overwriteModelType == OverwriteModelType.Classic)
{
context.ContentAnchorParser = new ContentAnchorParser(overwrite.Conceptual);
}
var transformed = _overwriteProcessor.Process(overwriteObject, schema, context) as IDictionary <string, object>;
if (_overwriteModelType == OverwriteModelType.Classic && !context.ContentAnchorParser.ContainsAnchor)
{
transformed["conceptual"] = context.ContentAnchorParser.Content;
}
// add SouceDetail back to transformed, in week type
if (overwrite.Documentation != null)
{
transformed[Constants.PropertyName.Documentation] = new Dictionary <string, object>
{
["remote"] = overwrite.Documentation.Remote == null ? null : new Dictionary <string, object>
{
["path"] = overwrite.Documentation.Remote.RelativePath,
["branch"] = overwrite.Documentation.Remote.RemoteBranch,
["repo"] = overwrite.Documentation.Remote.RemoteRepositoryUrl,
}
["path"] = overwrite.Documentation?.Path,
["startLine"] = overwrite.Documentation?.StartLine ?? 0,
["endLine"] = overwrite.Documentation?.EndLine ?? 0,
};
}
owModel.LinkToUids = owModel.LinkToUids.Union((context.UidLinkSources).Keys);
owModel.LinkToFiles = owModel.LinkToFiles.Union((context.FileLinkSources).Keys);
owModel.FileLinkSources = owModel.FileLinkSources.Merge(context.FileLinkSources);
owModel.UidLinkSources = owModel.UidLinkSources.Merge(context.UidLinkSources);
owModel.Uids = owModel.Uids.AddRange(context.Uids);
owModel.Properties.XRefSpecs = context.XRefSpecs;
owModel.Properties.ExternalXRefSpecs = context.ExternalXRefSpecs;
foreach (var d in context.Dependency)
{
_host.ReportDependencyTo(owModel, d, DependencyTypeName.Include);
}
return(transformed);
}
public void Build(FileModel model, IHostService host)
{
//.....
host.ReportDependencyTo(model, "uid", DependencyItemSourceType.Uid, "ref");
}