public Manifest Build(DocumentBuildParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.OutputBaseDir == null)
{
throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
}
if (parameters.Files == null)
{
throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
}
if (parameters.MaxParallelism <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
if (parameters.Metadata == null)
{
parameters.Metadata = ImmutableDictionary<string, object>.Empty;
}
return BuildCore(parameters);
}
private IEnumerable <IDocumentProcessor> LoadSchemaDrivenDocumentProcessors(DocumentBuildParameters parameter)
{
using (var resource = parameter?.TemplateManager?.CreateTemplateResource())
{
if (resource == null || resource.IsEmpty)
{
yield break;
}
foreach (var pair in resource.GetResourceStreams(@"^schemas/.*\.schema\.json"))
{
var fileName = Path.GetFileName(pair.Key);
using (new LoggerFileScope(fileName))
{
using (var stream = pair.Value)
{
using (var sr = new StreamReader(stream))
{
var schema = DocumentSchema.Load(sr, fileName.Remove(fileName.Length - ".schema.json".Length));
var sdp = new SchemaDrivenDocumentProcessor(schema, new CompositionContainer(CompositionContainer.DefaultContainer));
Logger.LogVerbose($"\t{sdp.Name} with build steps ({string.Join(", ", from bs in sdp.BuildSteps orderby bs.BuildOrder select bs.Name)})");
yield return(sdp);
}
}
}
}
}
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
var(models, invalidFiles) = LoadModels(files, parameters, processor);
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
models,
parameters.VersionName,
parameters.VersionDir,
parameters.LruSize,
parameters.GroupInfo,
new BuildParameters(parameters.TagParameters))
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
InvalidSourceFiles = invalidFiles.ToImmutableList(),
};
return(hostService);
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
files == null
? Enumerable.Empty <FileModel>()
: from file in files
select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model
where model != null
select model,
parameters.VersionName,
parameters.VersionDir,
parameters.LruSize,
parameters.GroupInfo)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
};
return(hostService);
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable<IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable<FileAndType> files)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
files == null
? new FileModel[0]
: from file in files
select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
};
return hostService;
}
public Manifest Build(DocumentBuildParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.OutputBaseDir == null)
{
throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
}
if (parameters.Files == null)
{
throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
}
if (parameters.MaxParallelism <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
if (parameters.Metadata == null)
{
parameters.Metadata = ImmutableDictionary <string, object> .Empty;
}
return(BuildCore(parameters));
}
private static string ComputeConfigHash(DocumentBuildParameters parameter)
{
return(JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalCheckPropertiesResolver()
}).GetMd5String());
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
using (new LoggerPhaseScope(pair.processor.Name, true))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return(hostService);
}
}
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files
.Where(s => s.Key == null)
.SelectMany(s => s)
.Where(s => s.Type != DocumentType.Overwrite &&
!s.File.EndsWith(".yaml.md", StringComparison.OrdinalIgnoreCase) &&
!s.File.EndsWith(".yml.md", StringComparison.OrdinalIgnoreCase))
.ToList();
if (notToHandleItems.Count > 0)
{
Logger.LogWarning(
$"Unable to handle following files: {notToHandleItems.Select(s => s.File).ToDelimitedString()}. Do they miss `YamlMime` as the first line of file, e.g.: `### YamlMime:ManagedReference`?",
code: WarningCodes.Build.UnknownContentType);
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
where item != null && item.Any(s => s.Type != DocumentType.Overwrite) // when normal file exists then processing is needed
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw ex.GetBaseException();
}
}
private List <IDocumentProcessor> LoadSchemaDrivenDocumentProcessors(DocumentBuildParameters parameter)
{
using (new LoggerPhaseScope(nameof(LoadSchemaDrivenDocumentProcessors)))
{
var result = new List <IDocumentProcessor>();
SchemaValidateService.RegisterLicense(parameter.SchemaLicense);
using (var resource = parameter?.TemplateManager?.CreateTemplateResource())
{
if (resource == null || resource.IsEmpty)
{
return(result);
}
var markdigMarkdownService = CreateMarkdigMarkdownService(parameter);
foreach (var pair in resource.GetResourceStreams(@"^schemas/.*\.schema\.json"))
{
var fileName = Path.GetFileName(pair.Key);
using (new LoggerFileScope(fileName))
{
using (var stream = pair.Value)
{
using (var sr = new StreamReader(stream))
{
DocumentSchema schema;
try
{
schema = DocumentSchema.Load(sr, fileName.Remove(fileName.Length - ".schema.json".Length));
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
var sdp = new SchemaDrivenDocumentProcessor(
schema,
new CompositionContainer(CompositionContainer.DefaultContainer),
markdigMarkdownService,
new FolderRedirectionManager(parameter.OverwriteFragmentsRedirectionRules));
Logger.LogVerbose($"\t{sdp.Name} with build steps ({string.Join(", ", from bs in sdp.BuildSteps orderby bs.BuildOrder select bs.Name)})");
result.Add(sdp);
}
}
}
}
}
if (result.Count > 0)
{
Logger.LogInfo($"{result.Count} schema driven document processor plug-in(s) loaded.");
Processors = Processors.Union(result);
}
return(result);
}
}
private void SaveDependency(DocumentBuildContext context, DocumentBuildParameters parameters)
{
var vbi = _currentBuildInfo.Versions.Find(v => v.VersionName == parameters.VersionName);
vbi.Dependency = Path.GetRandomFileName();
using (var writer = File.CreateText(
Path.Combine(IntermediateFolder, vbi.Dependency)))
{
context.DependencyGraph.Save(writer);
}
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.File);
}
Logger.LogWarning(sb.ToString());
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw new DocfxException(ex.InnerException?.Message, ex);
}
}
private Dictionary <string, FileAttributeItem> ComputeFileAttributes(DocumentBuildParameters parameters)
{
return((from f in parameters.Files.EnumerateFiles()
let fileKey = ((RelativePath)f.File).GetPathFromWorkingFolder().ToString()
select new FileAttributeItem
{
File = fileKey,
LastModifiedTime = File.GetLastWriteTimeUtc(f.FullPath),
MD5 = File.ReadAllText(f.FullPath).GetMd5String(),
}).ToDictionary(a => a.File));
}
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary <string, string> tokens)
{
return(MarkdownServiceProvider.CreateMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
TemplateDir = parameters.TemplateDir,
Extensions = parameters.MarkdownEngineParameters,
Tokens = tokens,
}));
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
return(from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new HostService(
parameters.Files.DefaultBaseDir,
item == null
? new FileModel[0]
: from file in item
select Load(processor, parameters.Metadata, parameters.FileMetadata, file.file) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
});
}
private static string GetXrefMapFileNameWithGroup(DocumentBuildParameters parameters)
{
if (!string.IsNullOrEmpty(parameters.GroupInfo?.Name))
{
return(parameters.GroupInfo.Name + "." + XRefMapFileName);
}
if (!string.IsNullOrEmpty(parameters.VersionName))
{
return(Uri.EscapeDataString(parameters.VersionName) + "." + XRefMapFileName);
}
return(XRefMapFileName);
}
public static ImmutableList <FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
from file in parameters.Files.EnumerateFiles()
select Load(processor, parameters.Metadata, parameters.FileMetadata, file)
into model
where model != null
select model);
BuildCore(processor, hostService, parameters.MaxParallelism);
return(hostService.Models);
}
public Manifest Build(DocumentBuildParameters parameters)
{
using (var builder = new SingleDocumentBuilder
{
Container = _container,
IntermediateFolder = IntermediateFolder,
MetadataValidators = MetadataValidators,
Processors = Processors
})
{
return(builder.Build(parameters));
}
}
private IEnumerable <IInputMetadataValidator> GetMetadataRules(DocumentBuildParameters parameter)
{
try
{
var mvb = MarkdownValidatorBuilder.Create(_container, parameter.Files.DefaultBaseDir, parameter.TemplateDir);
return(mvb.GetEnabledMetadataRules().ToList());
}
catch (Exception ex)
{
Logger.LogWarning($"Fail to init markdown style, details:{Environment.NewLine}{ex.ToString()}");
return(Enumerable.Empty <IInputMetadataValidator>());
}
}
internal Manifest BuildCore(DocumentBuildParameters parameter, IMarkdownServiceProvider markdownServiceProvider, BuildInfo currentBuildInfo, BuildInfo lastBuildInfo)
{
using var builder = new SingleDocumentBuilder
{
CurrentBuildInfo = currentBuildInfo,
LastBuildInfo = lastBuildInfo,
IntermediateFolder = _intermediateFolder,
MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(),
Processors = Processors,
MarkdownServiceProvider = markdownServiceProvider,
};
return(builder.Build(parameter));
}
private MarkdigMarkdownService CreateMarkdigMarkdownService(DocumentBuildParameters parameters)
{
var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(new DocumentBuildContext(parameters), parameters.MaxParallelism);
return(new MarkdigMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
TemplateDir = parameters.TemplateDir,
Extensions = parameters.MarkdownEngineParameters,
Tokens = templateProcessor?.Tokens?.ToImmutableDictionary(),
},
new CompositionContainer(CompositionContainer.DefaultContainer)));
}
public void PrepareMetadata(DocumentBuildParameters parameters)
{
foreach (var postProcessor in _postProcessors)
{
using (new LoggerPhaseScope(postProcessor.ContractName))
{
parameters.Metadata = postProcessor.Processor.PrepareMetadata(parameters.Metadata);
if (parameters.Metadata == null)
{
throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null metadata");
}
}
}
}
private MarkdigMarkdownService CreateMarkdigMarkdownService(DocumentBuildParameters parameters)
{
var resourceProvider = parameters.TemplateManager?.CreateTemplateResource();
return(new MarkdigMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
TemplateDir = parameters.TemplateDir,
Extensions = parameters.MarkdownEngineParameters,
Tokens = TemplateProcessorUtility.LoadTokens(resourceProvider)?.ToImmutableDictionary(),
},
new CompositionContainer(CompositionContainer.DefaultContainer)));
}
private static IEnumerable <InnerBuildContext> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
return(from item in toHandleItems.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new InnerBuildContext(
new HostService(
parameters.Files.DefaultBaseDir,
from file in item
select Load(item.Key, parameters.Metadata, parameters.FileMetadata, file.file)
into model
where model != null
select model)
{
MarkdownService = markdownService,
},
item.Key,
templateProcessor));
}
private void PrepareMetadata(DocumentBuildParameters parameters)
{
foreach (var postProcessor in _postProcessors)
{
using (new LoggerPhaseScope($"Prepare metadata in post processor {postProcessor.ContractName}", false))
using (new PerformanceScope($"Prepare metadata in post processor {postProcessor.ContractName}", LogLevel.Verbose))
{
parameters.Metadata = postProcessor.Processor.PrepareMetadata(parameters.Metadata);
if (parameters.Metadata == null)
{
throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null metadata");
}
}
}
}
internal Manifest BuildCore(DocumentBuildParameters parameter)
{
using (var builder = new SingleDocumentBuilder
{
Container = _container,
CurrentBuildInfo = _currentBuildInfo,
LastBuildInfo = _lastBuildInfo,
IntermediateFolder = _intermediateFolder,
MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(),
Processors = Processors,
})
{
return(builder.Build(parameter));
}
}
public override HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable<IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable<FileAndType> files)
{
if (ShouldProcessorTraceInfo(processor))
{
IncrementalContext.CreateProcessorInfo(processor);
}
var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files);
PostCreate(hs, files);
return hs;
}
/// <summary>
/// Export xref map file.
/// </summary>
private static void ExportXRefMap(DocumentBuildParameters parameters, DocumentBuildContext context)
{
Logger.LogVerbose("Exporting xref map...");
var xrefMap = new XRefMap();
xrefMap.References =
(from xref in context.XRefSpecMap.Values.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new XRefSpec(xref)
{
Href = ((RelativePath)context.FileMap[xref.Href]).RemoveWorkingFolder().ToString() + "#" + XRefDetails.GetHtmlId(xref.Uid),
}).ToList();
xrefMap.Sort();
YamlUtility.Serialize(
Path.Combine(parameters.OutputBaseDir, XRefMapFileName),
xrefMap);
Logger.LogInfo("XRef map exported.");
}
public override HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
if (ShouldProcessorTraceInfo(processor))
{
IncrementalContext.CreateProcessorInfo(processor);
}
var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files);
PostCreate(hs, files);
return(hs);
}
public static ImmutableList <FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
from file in parameters.Files.EnumerateFiles()
select Load(processor, parameters.Metadata, parameters.FileMetadata, file, false, null, null, null, null)
into model
where model != null
select model)
{
Processor = processor,
MarkdownService = markdownService,
DependencyGraph = new DependencyGraph(),
};
BuildCore(hostService, parameters.MaxParallelism);
return(hostService.Models);
}
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters)
{
var provider = (IMarkdownServiceProvider)_container.GetExport(
typeof(IMarkdownServiceProvider),
parameters.MarkdownEngineName);
if (provider == null)
{
Logger.LogError($"Unable to find markdown engine: {parameters.MarkdownEngineName}");
throw new DocfxException($"Unable to find markdown engine: {parameters.MarkdownEngineName}");
}
Logger.LogInfo($"Markdown engine is {parameters.MarkdownEngineName}");
return(provider.CreateMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
Extensions = parameters.MarkdownEngineParameters,
}));
}
private IEnumerable <FileModel> LoadModels(IEnumerable <FileAndType> files, DocumentBuildParameters parameters, IDocumentProcessor processor)
{
var models = new ConcurrentBag <FileModel>();
if (files == null)
{
return(models);
}
files.RunAll(file =>
{
var model = Load(processor, parameters.Metadata, parameters.FileMetadata, file);
if (model != null)
{
models.Add(model);
}
}, _context.MaxParallelism);
return(models);
}
public static ImmutableList<FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService)
{
var hostServiceCreator = new HostServiceCreator(null);
var hostService = hostServiceCreator.CreateHostService(
parameters,
null,
markdownService,
null,
processor,
parameters.Files.EnumerateFiles());
var phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(null),
new PostbuildPhaseHandler(null, null),
}
};
phaseProcessor.Process(new List<HostService> { hostService }, parameters.MaxParallelism);
return hostService.Models;
}
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary<string, string> tokens)
{
return MarkdownServiceProvider.CreateMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
TemplateDir = parameters.TemplateDir,
Extensions = parameters.MarkdownEngineParameters,
Tokens = tokens,
});
}
private Manifest BuildCore(DocumentBuildParameters parameters)
{
using (new LoggerPhaseScope(PhaseName, true))
{
Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages,
parameters.XRefMaps,
parameters.MaxParallelism,
parameters.Files.DefaultBaseDir,
parameters.VersionName,
parameters.ApplyTemplateSettings,
parameters.RootTocPath);
Logger.LogVerbose("Start building document...");
// Start building document...
List<HostService> hostServices = null;
IHostServiceCreator hostServiceCreator = null;
PhaseProcessor phaseProcessor = null;
try
{
using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor)
{
using (new LoggerPhaseScope("Prepare", true))
{
if (MarkdownService == null)
{
MarkdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary());
}
Prepare(
parameters,
context,
templateProcessor,
(MarkdownService as IHasIncrementalContext)?.GetIncrementalContextHash(),
out hostServiceCreator,
out phaseProcessor);
}
using (new LoggerPhaseScope("Load", true))
{
hostServices = GetInnerContexts(parameters, Processors, templateProcessor, hostServiceCreator).ToList();
}
BuildCore(phaseProcessor, hostServices, context);
return new Manifest
{
Files = context.ManifestItems.ToList(),
Homepages = GetHomepages(context),
XRefMap = ExportXRefMap(parameters, context),
SourceBasePath = StringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory)
};
}
}
finally
{
if (hostServices != null)
{
foreach (var item in hostServices)
{
Cleanup(item);
item.Dispose();
}
}
}
}
}
private IEnumerable<IInputMetadataValidator> GetMetadataRules(DocumentBuildParameters parameter)
{
try
{
var mvb = MarkdownValidatorBuilder.Create(_container, parameter.Files.DefaultBaseDir, parameter.TemplateDir);
return mvb.GetEnabledMetadataRules().ToList();
}
catch (Exception ex)
{
Logger.LogWarning($"Fail to init markdown style, details:{Environment.NewLine}{ex.ToString()}");
return Enumerable.Empty<IInputMetadataValidator>();
}
}
internal Manifest BuildCore(DocumentBuildParameters parameter)
{
using (var builder = new SingleDocumentBuilder
{
Container = _container,
CurrentBuildInfo = _currentBuildInfo,
LastBuildInfo = _lastBuildInfo,
IntermediateFolder = _intermediateFolder,
MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(),
Processors = Processors,
})
{
return builder.Build(parameter);
}
}
public static ImmutableList<FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
from file in parameters.Files.EnumerateFiles()
select Load(processor, parameters.Metadata, parameters.FileMetadata, file, false, null)
into model
where model != null
select model)
{
Processor = processor,
MarkdownService = markdownService,
DependencyGraph = new DependencyGraph(),
};
BuildCore(new List<HostService> { hostService }, parameters.MaxParallelism, null, null, null);
return hostService.Models;
}
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary<string, string> tokens)
{
var provider = (IMarkdownServiceProvider)Container.GetExport(
typeof(IMarkdownServiceProvider),
parameters.MarkdownEngineName);
if (provider == null)
{
Logger.LogError($"Unable to find markdown engine: {parameters.MarkdownEngineName}");
throw new DocfxException($"Unable to find markdown engine: {parameters.MarkdownEngineName}");
}
Logger.LogInfo($"Markdown engine is {parameters.MarkdownEngineName}");
return provider.CreateMarkdownService(
new MarkdownServiceParameters
{
BasePath = parameters.Files.DefaultBaseDir,
TemplateDir = parameters.TemplateDir,
Extensions = parameters.MarkdownEngineParameters,
Tokens = tokens,
});
}
private void PrepareMetadata(DocumentBuildParameters parameters)
{
foreach (var postProcessor in _postProcessors)
{
using (new LoggerPhaseScope($"Prepare metadata in post processor {postProcessor.ContractName}", false))
using (new PerformanceScope($"Prepare metadata in post processor {postProcessor.ContractName}", LogLevel.Verbose))
{
parameters.Metadata = postProcessor.Processor.PrepareMetadata(parameters.Metadata);
if (parameters.Metadata == null)
{
throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null metadata");
}
}
}
}
private Manifest BuildCore(DocumentBuildParameters parameters)
{
using (new LoggerPhaseScope(PhaseName, true))
{
Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages,
parameters.XRefMaps,
parameters.MaxParallelism,
parameters.Files.DefaultBaseDir);
if (ShouldTraceIncrementalInfo)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder);
Logger.RegisterListener(context.IncrementalBuildContext.CurrentBuildVersionInfo.BuildMessage.GetListener());
if (context.IncrementalBuildContext.CanVersionIncremental)
{
context.IncrementalBuildContext.LoadChanges();
Logger.LogVerbose($"Before expanding dependency before build, changes: {JsonUtility.Serialize(context.IncrementalBuildContext.ChangeDict, Formatting.Indented)}");
var dependencyGraph = context.IncrementalBuildContext.LastBuildVersionInfo.Dependency;
context.IncrementalBuildContext.ExpandDependency(dependencyGraph, d => dependencyGraph.DependencyTypes[d.Type].Phase == BuildPhase.Build || dependencyGraph.DependencyTypes[d.Type].TriggerBuild);
Logger.LogVerbose($"After expanding dependency before build, changes: {JsonUtility.Serialize(context.IncrementalBuildContext.ChangeDict, Formatting.Indented)}");
}
}
Logger.LogVerbose("Start building document...");
// Start building document...
List<HostService> hostServices = null;
try
{
using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor)
{
IMarkdownService markdownService;
using (new LoggerPhaseScope("CreateMarkdownService", true))
{
markdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary());
}
using (new LoggerPhaseScope("Load", true))
{
hostServices = GetInnerContexts(parameters, Processors, templateProcessor, markdownService, context).ToList();
}
var manifest = BuildCore(hostServices, context, parameters.VersionName).ToList();
// Use manifest from now on
using (new LoggerPhaseScope("UpdateContext", true))
{
UpdateContext(context);
}
// Run getOptions from Template
using (new LoggerPhaseScope("FeedOptions", true))
{
FeedOptions(manifest, context);
}
// Template can feed back xref map, actually, the anchor # location can only be determined in template
using (new LoggerPhaseScope("FeedXRefMap", true))
{
FeedXRefMap(manifest, context);
}
using (new LoggerPhaseScope("UpdateHref", true))
{
UpdateHref(manifest, context);
}
// Afterwards, m.Item.Model.Content is always IDictionary
using (new LoggerPhaseScope("ApplySystemMetadata", true))
{
ApplySystemMetadata(manifest, context);
}
// Register global variables after href are all updated
IDictionary<string, object> globalVariables;
using (new LoggerPhaseScope("FeedGlobalVariables", true))
{
globalVariables = FeedGlobalVariables(templateProcessor.Tokens, manifest, context);
}
// processor to add global variable to the model
foreach (var m in templateProcessor.Process(manifest.Select(s => s.Item).ToList(), context, parameters.ApplyTemplateSettings, globalVariables))
{
context.ManifestItems.Add(m);
}
return new Manifest
{
Files = context.ManifestItems.ToList(),
Homepages = GetHomepages(context),
XRefMap = ExportXRefMap(parameters, context),
SourceBasePath = TypeForwardedToStringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory)
};
}
}
finally
{
if (hostServices != null)
{
foreach (var item in hostServices)
{
Cleanup(item);
item.Dispose();
}
}
}
}
}
private IEnumerable<HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable<IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
DocumentBuildContext context)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null }
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var incrementalContext = context.IncrementalBuildContext;
var processorSupportIncremental = IsProcessorSupportIncremental(pair.processor);
bool processorCanIncremental = processorSupportIncremental;
if (processorSupportIncremental)
{
incrementalContext.CreateProcessorInfo(pair.processor);
processorCanIncremental = incrementalContext.CanProcessorIncremental(pair.processor);
}
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
pair.item == null
? new FileModel[0]
: from file in pair.item
select Load(pair.processor, parameters.Metadata, parameters.FileMetadata, file.file, processorCanIncremental, context) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = pair.processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
ShouldTraceIncrementalInfo = processorSupportIncremental,
CanIncrementalBuild = processorCanIncremental,
};
if (ShouldTraceIncrementalInfo)
{
using (new LoggerPhaseScope("ReportModelLoadInfo", true))
{
var allFiles = pair.item?.Select(f => f.file.File) ?? new string[0];
var loadedFiles = hostService.Models.Select(m => m.FileAndType.File);
incrementalContext.ReportModelLoadInfo(hostService, allFiles.Except(loadedFiles), null);
incrementalContext.ReportModelLoadInfo(hostService, loadedFiles, BuildPhase.PreBuild);
}
}
yield return hostService;
}
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
};
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null))
{
builder.Build(parameters);
}
}
public void Build(DocumentBuildParameters parameter)
{
Build(new DocumentBuildParameters[] { parameter }, parameter.OutputBaseDir);
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary<string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null))
{
builder.Build(parameters);
}
}
private IEnumerable<HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable<IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null }
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return hostService;
}
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
/// <summary>
/// Export xref map file.
/// </summary>
private static string ExportXRefMap(DocumentBuildParameters parameters, DocumentBuildContext context)
{
Logger.LogVerbose("Exporting xref map...");
var xrefMap = new XRefMap();
xrefMap.References =
(from xref in context.XRefSpecMap.Values.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new XRefSpec(xref)
{
Href = ((RelativePath)context.FileMap[UriUtility.GetNonFragment(xref.Href)]).RemoveWorkingFolder() + UriUtility.GetFragment(xref.Href)
}).ToList();
xrefMap.Sort();
string xrefMapFileNameWithVersion = string.IsNullOrEmpty(parameters.VersionName) ?
XRefMapFileName :
parameters.VersionName + "." + XRefMapFileName;
YamlUtility.Serialize(
Path.Combine(parameters.OutputBaseDir, xrefMapFileNameWithVersion),
xrefMap,
YamlMime.XRefMap);
Logger.LogInfo("XRef map exported.");
return xrefMapFileNameWithVersion;
}
