private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
var(models, invalidFiles) = LoadModels(files, parameters, processor);
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
models,
parameters.VersionName,
parameters.VersionDir,
parameters.LruSize,
parameters.GroupInfo,
new BuildParameters(parameters.TagParameters))
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
InvalidSourceFiles = invalidFiles.ToImmutableList(),
};
return(hostService);
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable<IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable<FileAndType> files)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
files == null
? new FileModel[0]
: from file in files
select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
};
return hostService;
}
public virtual HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
files == null
? Enumerable.Empty <FileModel>()
: from file in files
select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model
where model != null
select model,
parameters.VersionName,
parameters.VersionDir,
parameters.LruSize,
parameters.GroupInfo)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = metadataValidator?.ToImmutableList(),
ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor),
CanIncrementalBuild = CanProcessorIncremental(processor),
};
return(hostService);
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files
.Where(s => s.Key == null)
.SelectMany(s => s)
.Where(s => s.Type != DocumentType.Overwrite &&
!s.File.EndsWith(".yaml.md", StringComparison.OrdinalIgnoreCase) &&
!s.File.EndsWith(".yml.md", StringComparison.OrdinalIgnoreCase))
.ToList();
if (notToHandleItems.Count > 0)
{
Logger.LogWarning(
$"Unable to handle following files: {notToHandleItems.Select(s => s.File).ToDelimitedString()}. Do they miss `YamlMime` as the first line of file, e.g.: `### YamlMime:ManagedReference`?",
code: WarningCodes.Build.UnknownContentType);
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
where item != null && item.Any(s => s.Type != DocumentType.Overwrite) // when normal file exists then processing is needed
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw ex.GetBaseException();
}
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
using (new LoggerPhaseScope(pair.processor.Name, true))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return(hostService);
}
}
}
private List <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
group file by p).ToList();
var toHandleItems = files.Where(s => s.Key != null);
var notToHandleItems = files.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.File);
}
Logger.LogWarning(sb.ToString());
}
try
{
return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
join item in toHandleItems.AsParallel() on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select LoggerPhaseScope.WithScope(
processor.Name,
LogLevel.Verbose,
() => creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
processor,
item)
)).ToList());
}
catch (AggregateException ex)
{
throw new DocfxException(ex.InnerException?.Message, ex);
}
}
public ManifestProcessor(List <ManifestItemWithContext> manifestWithContext, DocumentBuildContext context, TemplateProcessor templateProcessor)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
_templateProcessor = templateProcessor ?? throw new ArgumentNullException(nameof(templateProcessor));
_manifestWithContext = manifestWithContext ?? throw new ArgumentNullException(nameof(manifestWithContext));
// E.g. we can set TOC model to be globally shared by every data model
// Make sure it is single thread
_globalMetadata = _templateProcessor.Tokens?.ToDictionary(pair => pair.Key, pair => (object)pair.Value)
?? new Dictionary <string, object>();
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
return(from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new HostService(
parameters.Files.DefaultBaseDir,
item == null
? new FileModel[0]
: from file in item
select Load(processor, parameters.Metadata, parameters.FileMetadata, file.file) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
});
}
public static List <TemplateManifestItem> Transform(TemplateProcessor processor, List <ManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings)
{
if (settings.Options == ApplyTemplateOptions.ExportRawModel || processor == null)
{
ExportRawModel(manifest, settings);
return(null);
}
using (new LoggerPhaseScope("Apply Templates"))
{
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
processor.ProcessDependencies(settings.OutputFolder);
if (processor.IsEmpty)
{
Logger.LogWarning("No template is found.");
ExportRawModel(manifest, settings);
return(null);
}
Logger.LogVerbose("Start applying template...");
var outputDirectory = context.BuildOutputFolder;
var templateManifest = processor.Transform(manifest, context, settings);
if (!settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
if (templateManifest.Count > 0)
{
// Save manifest from template
// TODO: Keep .manifest for backward-compatability, will remove next sprint
var manifestPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ObsoleteManifestFileName);
JsonUtility.Serialize(manifestPath, templateManifest);
// Logger.LogInfo($"Manifest file saved to {manifestPath}. NOTE: This file is out-of-date and will be removed in version 1.8, if you rely on this file, please change to use {Constants.ManifestFileName} instead.");
var manifestJsonPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ManifestFileName);
var toc = context.GetTocInfo();
var manifestObject = GenerateManifest(context, templateManifest);
JsonUtility.Serialize(manifestJsonPath, manifestObject);
Logger.LogInfo($"Manifest file saved to {manifestJsonPath}.");
}
return(templateManifest);
}
}
private static IEnumerable <InnerBuildContext> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
return(from item in toHandleItems.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new InnerBuildContext(
new HostService(
parameters.Files.DefaultBaseDir,
from file in item
select Load(item.Key, parameters.Metadata, parameters.FileMetadata, file.file)
into model
where model != null
select model)
{
MarkdownService = markdownService,
},
item.Key,
templateProcessor));
}
private void ProcessUnloadedTemplateDependency(IEnumerable <HostService> hostServices)
{
var loaded = Context.ManifestItems;
var unloaded = GetUnloadedManifestItems(hostServices);
var types = new HashSet <string>(unloaded.Select(m => m.DocumentType).Except(loaded.Select(m => m.DocumentType)));
if (types.Count > 0)
{
TemplateProcessor.ProcessDependencies(types, Context.ApplyTemplateSettings);
}
foreach (var m in unloaded)
{
Context.ManifestItems.Add(m);
}
}
public override HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable<IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable<FileAndType> files)
{
if (ShouldProcessorTraceInfo(processor))
{
IncrementalContext.CreateProcessorInfo(processor);
}
var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files);
PostCreate(hs, files);
return hs;
}
public override HostService CreateHostService(
DocumentBuildParameters parameters,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
IEnumerable <IInputMetadataValidator> metadataValidator,
IDocumentProcessor processor,
IEnumerable <FileAndType> files)
{
if (ShouldProcessorTraceInfo(processor))
{
IncrementalContext.CreateProcessorInfo(processor);
}
var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files);
PostCreate(hs, files);
return(hs);
}
public ManifestProcessor(IEnumerable<HostService> hostServices, DocumentBuildContext context, TemplateProcessor templateProcessor)
{
if (hostServices == null)
{
throw new ArgumentNullException(nameof(hostServices));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (templateProcessor == null)
{
throw new ArgumentNullException(nameof(templateProcessor));
}
_context = context;
_templateProcessor = templateProcessor;
Init(hostServices);
}
public ManifestProcessor(List <HostService> hostServices, DocumentBuildContext context, TemplateProcessor templateProcessor)
{
if (hostServices == null)
{
throw new ArgumentNullException(nameof(hostServices));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (templateProcessor == null)
{
throw new ArgumentNullException(nameof(templateProcessor));
}
_context = context;
_templateProcessor = templateProcessor;
Init(hostServices);
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
using (new LoggerPhaseScope("CreateIncrementalBuildContext", false))
using (new PerformanceScope("CreateIncrementalBuildContext", LogLevel.Verbose))
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
}
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new CompilePhaseHandler(context).WithIncremental(),
new LinkPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new CompilePhaseHandler(context),
new LinkPhaseHandler(context, templateProcessor),
}
};
}
}
private void ProcessUnloadedTemplateDependency(IEnumerable <HostService> hostServices, int maxParallelism)
{
var loaded = Context.ManifestItems;
IEnumerable <ManifestItem> unloaded;
using (new LoggerPhaseScope("GetUnloadedManifestItems", LogLevel.Verbose))
{
unloaded = GetUnloadedManifestItems(hostServices, maxParallelism);
}
var types = new HashSet <string>(unloaded.Select(m => m.DocumentType).Except(loaded.Select(m => m.DocumentType)));
if (types.Count > 0)
{
using (new LoggerPhaseScope("ProcessDependencies", LogLevel.Verbose))
{
TemplateProcessor.ProcessDependencies(types, Context.ApplyTemplateSettings);
}
}
foreach (var m in unloaded)
{
Context.ManifestItems.Add(m);
}
}
public void Build(DocumentBuildParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.OutputBaseDir == null)
{
throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
}
if (parameters.Files == null)
{
throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
}
if (parameters.MaxParallelism <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
if (parameters.Metadata == null)
{
parameters.Metadata = ImmutableDictionary <string, object> .Empty;
}
using (new LoggerPhaseScope(PhaseName))
{
Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism.ToString()}.");
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Environment.CurrentDirectory, parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages,
parameters.MaxParallelism);
Logger.LogVerbose("Start building document...");
var innerContexts = Enumerable.Empty <InnerBuildContext>();
try
{
using (var processor = parameters.TemplateManager?.GetTemplateProcessor(parameters.MaxParallelism))
{
innerContexts = GetInnerContexts(parameters, Processors, processor).ToList();
var manifest = new List <ManifestItemWithContext>();
foreach (var item in innerContexts)
{
manifest.AddRange(BuildCore(item, context));
}
// Use manifest from now on
UpdateContext(context);
UpdateHref(manifest, context);
var generatedManifest = TemplateProcessor.Transform(processor, manifest.Select(s => s.Item).ToList(), context, parameters.ApplyTemplateSettings);
// todo : move to plugin.
object value;
if (parameters.Metadata.TryGetValue("_enableSearch", out value))
{
var isSearchable = value as bool?;
if (isSearchable.HasValue && isSearchable.Value)
{
ExtractSearchData.ExtractSearchIndexFromHtml.GenerateFile(generatedManifest, parameters.OutputBaseDir);
}
}
Logger.LogInfo($"Building {manifest.Count} file(s) completed.");
}
}
finally
{
foreach (var item in innerContexts)
{
if (item.HostService != null)
{
Cleanup(item.HostService);
item.HostService.Dispose();
}
}
}
}
}
public LinkPhaseHandler(DocumentBuildContext context, TemplateProcessor templateProcessor)
{
Context = context;
TemplateProcessor = templateProcessor;
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
DocumentBuildContext context)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
var pairs = from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
canProcessorIncremental = CanProcessorIncremental(processor, parameters.VersionName),
};
// load last xrefspecmap and manifestitems
BuildVersionInfo lbvi = LastBuildInfo?.Versions.SingleOrDefault(v => v.VersionName == parameters.VersionName);
var lastXRefSpecMap = lbvi?.XRefSpecMap;
var lastManifest = lbvi?.Manifest;
var lastDependencyGraph = lbvi?.Dependency;
return(from pair in pairs.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new HostService(
parameters.Files.DefaultBaseDir,
pair.item == null
? new FileModel[0]
: from file in pair.item
select Load(pair.processor, parameters.Metadata, parameters.FileMetadata, file.file, pair.canProcessorIncremental, lastXRefSpecMap, lastManifest, lastDependencyGraph, context) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = pair.processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
});
}
public ManifestProcessor(List <ManifestItemWithContext> manifestWithContext, DocumentBuildContext context, TemplateProcessor templateProcessor)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
_templateProcessor = templateProcessor ?? throw new ArgumentNullException(nameof(templateProcessor));
_manifestWithContext = manifestWithContext ?? throw new ArgumentNullException(nameof(manifestWithContext));
}
private IEnumerable <HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable <IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
DocumentBuildContext context)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> {
null
}
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var incrementalContext = context.IncrementalBuildContext;
var processorSupportIncremental = IsProcessorSupportIncremental(pair.processor);
bool processorCanIncremental = processorSupportIncremental;
if (processorSupportIncremental)
{
incrementalContext.CreateProcessorInfo(pair.processor);
processorCanIncremental = incrementalContext.CanProcessorIncremental(pair.processor);
}
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
pair.item == null
? new FileModel[0]
: from file in pair.item
select Load(pair.processor, parameters.Metadata, parameters.FileMetadata, file.file, processorCanIncremental, context) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = pair.processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
ShouldTraceIncrementalInfo = processorSupportIncremental,
CanIncrementalBuild = processorCanIncremental,
};
if (ShouldTraceIncrementalInfo)
{
using (new LoggerPhaseScope("ReportModelLoadInfo", true))
{
var allFiles = pair.item?.Select(f => f.file.File) ?? new string[0];
var loadedFiles = hostService.Models.Select(m => m.FileAndType.File);
incrementalContext.ReportModelLoadInfo(hostService, allFiles.Except(loadedFiles), null);
incrementalContext.ReportModelLoadInfo(hostService, loadedFiles, BuildPhase.PreBuild);
}
}
yield return(hostService);
}
}
private void Prepare(
DocumentBuildParameters parameters,
DocumentBuildContext context,
TemplateProcessor templateProcessor,
string markdownServiceContextHash,
out IHostServiceCreator hostServiceCreator,
out PhaseProcessor phaseProcessor)
{
if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument)
{
context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash);
hostServiceCreator = new HostServiceCreatorWithIncremental(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context).WithIncremental(),
new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(),
}
};
}
else
{
hostServiceCreator = new HostServiceCreator(context);
phaseProcessor = new PhaseProcessor
{
Handlers =
{
new PrebuildBuildPhaseHandler(context),
new PostbuildPhaseHandler(context, templateProcessor),
}
};
}
}
private IEnumerable<HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable<IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IHostServiceCreator creator)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null }
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var hostService = creator.CreateHostService(
parameters,
templateProcessor,
MarkdownService,
MetadataValidators,
pair.processor,
pair.item?.Select(f => f.file));
yield return hostService;
}
}
public InnerBuildContext(HostService hostService, IDocumentProcessor processor, TemplateProcessor templateProcessor)
{
HostService = hostService;
Processor = processor;
TemplateProcessor = templateProcessor;
}
private IEnumerable<HostService> GetInnerContexts(
DocumentBuildParameters parameters,
IEnumerable<IDocumentProcessor> processors,
TemplateProcessor templateProcessor,
IMarkdownService markdownService,
DocumentBuildContext context)
{
var k = from fileItem in (
from file in parameters.Files.EnumerateFiles()
from p in (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupported
group processor by priority into ps
orderby ps.Key descending
select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null }
select new { file, p })
group fileItem by fileItem.p;
var toHandleItems = k.Where(s => s.Key != null);
var notToHandleItems = k.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.file.File);
}
Logger.LogWarning(sb.ToString());
}
// todo : revert until PreProcessor ready
foreach (var pair in (from processor in processors
join item in toHandleItems on processor equals item.Key into g
from item in g.DefaultIfEmpty()
select new
{
processor,
item,
}).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism))
{
var incrementalContext = context.IncrementalBuildContext;
var processorSupportIncremental = IsProcessorSupportIncremental(pair.processor);
bool processorCanIncremental = processorSupportIncremental;
if (processorSupportIncremental)
{
incrementalContext.CreateProcessorInfo(pair.processor);
processorCanIncremental = incrementalContext.CanProcessorIncremental(pair.processor);
}
var hostService = new HostService(
parameters.Files.DefaultBaseDir,
pair.item == null
? new FileModel[0]
: from file in pair.item
select Load(pair.processor, parameters.Metadata, parameters.FileMetadata, file.file, processorCanIncremental, context) into model
where model != null
select model)
{
MarkdownService = markdownService,
Processor = pair.processor,
Template = templateProcessor,
Validators = MetadataValidators.ToImmutableList(),
ShouldTraceIncrementalInfo = processorSupportIncremental,
CanIncrementalBuild = processorCanIncremental,
};
if (ShouldTraceIncrementalInfo)
{
using (new LoggerPhaseScope("ReportModelLoadInfo", true))
{
var allFiles = pair.item?.Select(f => f.file.File) ?? new string[0];
var loadedFiles = hostService.Models.Select(m => m.FileAndType.File);
incrementalContext.ReportModelLoadInfo(hostService, allFiles.Except(loadedFiles), null);
incrementalContext.ReportModelLoadInfo(hostService, loadedFiles, BuildPhase.PreBuild);
}
}
yield return hostService;
}
}
