private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files
.Where(s => s.Key == null)
.SelectMany(s => s)
.Where(s => s.Type != DocumentType.Overwrite &&
!s.File.EndsWith(".yaml.md", StringComparison.OrdinalIgnoreCase) &&
!s.File.EndsWith(".yml.md", StringComparison.OrdinalIgnoreCase))
.ToList();
if (notToHandleItems.Count > 0)
{
Logger.LogWarning(
$"Unable to handle following files: {notToHandleItems.Select(s => s.File).ToDelimitedString()}. Do they miss `YamlMime` as the first line of file, e.g.: `### YamlMime:ManagedReference`?",
code: WarningCodes.Build.UnknownContentType);
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
where item != null && item.Any(s => s.Type != DocumentType.Overwrite) // when normal file exists then processing is needed
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw ex.GetBaseException();
}
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
using (new LoggerPhaseScope(pair.processor.Name, true))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return(hostService);
}
}
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.File);
}
Logger.LogWarning(sb.ToString());
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw new DocfxException(ex.InnerException?.Message, ex);
}
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
using (new LoggerPhaseScope("CreateIncrementalBuildContext", false))
using (new PerformanceScope("CreateIncrementalBuildContext", LogLevel.Verbose))
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
}
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new CompilePhaseHandler(context).WithIncremental(),
new LinkPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new CompilePhaseHandler(context),
new LinkPhaseHandler(context, templateProcessor),
}
};
}
}
private Manifest BuildCore(DocumentBuildParameters parameters)
{
using (new LoggerPhaseScope(PhaseName, LogLevel.Verbose))
{
Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(parameters);
Logger.LogVerbose("Start building document...");
// Start building document...
List <HostService> hostServices = null;
IHostServiceCreator hostServiceCreator = null;
PhaseProcessor phaseProcessor = null;
try
{
using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism)
?? new TemplateProcessor(new EmptyResourceReader(), context, 16))
{
using (new LoggerPhaseScope("Prepare", LogLevel.Verbose))
{
if (MarkdownService == null)
{
using (new LoggerPhaseScope("CreateMarkdownService", LogLevel.Verbose))
{
MarkdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary());
context.MarkdownService = MarkdownService;
}
}
Prepare(
parameters,
context,
templateProcessor,
(MarkdownService as IHasIncrementalContext)?.GetIncrementalContextHash(),
out hostServiceCreator,
out phaseProcessor);
}
using (new LoggerPhaseScope("Load", LogLevel.Verbose))
{
hostServices = GetInnerContexts(parameters, Processors, templateProcessor, hostServiceCreator);
}
BuildCore(phaseProcessor, hostServices, context);
var manifest = new Manifest(context.ManifestItems)
{
Homepages = GetHomepages(context),
XRefMap = ExportXRefMap(parameters, context),
SourceBasePath = StringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory),
IncrementalInfo = context.IncrementalBuildContext != null ? new List <IncrementalInfo> {
context.IncrementalBuildContext.IncrementalInfo
} : null,
VersionInfo = string.IsNullOrEmpty(context.VersionName) ?
new Dictionary <string, VersionInfo>() :
new Dictionary <string, VersionInfo>
{
{
context.VersionName,
new VersionInfo {
VersionFolder = context.VersionFolder
}
}
}
};
manifest.Groups = new List <ManifestGroupInfo>
{
new ManifestGroupInfo(parameters.GroupInfo)
{
XRefmap = (string)manifest.XRefMap
}
};
return(manifest);
}
}
finally
{
if (hostServices != null)
{
foreach (var item in hostServices)
{
Cleanup(item);
item.Dispose();
}
}
}
}
}
private Manifest BuildCore(DocumentBuildParameters parameters)
{
using (new LoggerPhaseScope(PhaseName, LogLevel.Verbose))
{
Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages,
parameters.XRefMaps,
parameters.MaxParallelism,
parameters.Files.DefaultBaseDir,
parameters.VersionName,
parameters.ApplyTemplateSettings,
parameters.RootTocPath,
parameters.VersionDir);
Logger.LogVerbose("Start building document...");
// Start building document...
List <HostService> hostServices = null;
IHostServiceCreator hostServiceCreator = null;
PhaseProcessor phaseProcessor = null;
try
{
using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor)
{
using (new LoggerPhaseScope("Prepare", LogLevel.Verbose))
{
if (MarkdownService == null)
{
using (new LoggerPhaseScope("CreateMarkdownService", LogLevel.Verbose))
{
MarkdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary());
context.MarkdownService = MarkdownService;
}
}
Prepare(
parameters,
context,
templateProcessor,
(MarkdownService as IHasIncrementalContext)?.GetIncrementalContextHash(),
out hostServiceCreator,
out phaseProcessor);
}
using (new LoggerPhaseScope("Load", LogLevel.Verbose))
{
hostServices = GetInnerContexts(parameters, Processors, templateProcessor, hostServiceCreator);
}
BuildCore(phaseProcessor, hostServices, context);
return(new Manifest(context.ManifestItems)
{
Homepages = GetHomepages(context),
XRefMap = ExportXRefMap(parameters, context),
SourceBasePath = StringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory),
IncrementalInfo = context.IncrementalBuildContext != null ? new List <IncrementalInfo> {
context.IncrementalBuildContext.IncrementalInfo
} : null,
VersionInfo = string.IsNullOrEmpty(context.VersionName) ?
new Dictionary <string, VersionInfo>():
new Dictionary <string, VersionInfo>
{
{
context.VersionName,
new VersionInfo {
VersionFolder = context.VersionFolder
}
}
}
});
}
}
finally
{
if (hostServices != null)
{
foreach (var item in hostServices)
{
Cleanup(item);
item.Dispose();
}
}
}
}
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
private IEnumerable<HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable<IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null }
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return hostService;
}
}
