public Manifest Build(DocumentBuildParameters parameters) { if (parameters == null) { throw new ArgumentNullException(nameof(parameters)); } if (parameters.OutputBaseDir == null) { throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir)); } if (parameters.Files == null) { throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files)); } if (parameters.MaxParallelism <= 0) { parameters.MaxParallelism = Environment.ProcessorCount; } if (parameters.Metadata == null) { parameters.Metadata = ImmutableDictionary<string, object>.Empty; } return BuildCore(parameters); }
private IEnumerable <IDocumentProcessor> LoadSchemaDrivenDocumentProcessors(DocumentBuildParameters parameter) { using (var resource = parameter?.TemplateManager?.CreateTemplateResource()) { if (resource == null || resource.IsEmpty) { yield break; } foreach (var pair in resource.GetResourceStreams(@"^schemas/.*\.schema\.json")) { var fileName = Path.GetFileName(pair.Key); using (new LoggerFileScope(fileName)) { using (var stream = pair.Value) { using (var sr = new StreamReader(stream)) { var schema = DocumentSchema.Load(sr, fileName.Remove(fileName.Length - ".schema.json".Length)); var sdp = new SchemaDrivenDocumentProcessor(schema, new CompositionContainer(CompositionContainer.DefaultContainer)); Logger.LogVerbose($"\t{sdp.Name} with build steps ({string.Join(", ", from bs in sdp.BuildSteps orderby bs.BuildOrder select bs.Name)})"); yield return(sdp); } } } } } }
public virtual HostService CreateHostService( DocumentBuildParameters parameters, TemplateProcessor templateProcessor, IMarkdownService markdownService, IEnumerable <IInputMetadataValidator> metadataValidator, IDocumentProcessor processor, IEnumerable <FileAndType> files) { var(models, invalidFiles) = LoadModels(files, parameters, processor); var hostService = new HostService( parameters.Files.DefaultBaseDir, models, parameters.VersionName, parameters.VersionDir, parameters.LruSize, parameters.GroupInfo, new BuildParameters(parameters.TagParameters)) { MarkdownService = markdownService, Processor = processor, Template = templateProcessor, Validators = metadataValidator?.ToImmutableList(), ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor), CanIncrementalBuild = CanProcessorIncremental(processor), InvalidSourceFiles = invalidFiles.ToImmutableList(), }; return(hostService); }
public virtual HostService CreateHostService( DocumentBuildParameters parameters, TemplateProcessor templateProcessor, IMarkdownService markdownService, IEnumerable <IInputMetadataValidator> metadataValidator, IDocumentProcessor processor, IEnumerable <FileAndType> files) { var hostService = new HostService( parameters.Files.DefaultBaseDir, files == null ? Enumerable.Empty <FileModel>() : from file in files select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model where model != null select model, parameters.VersionName, parameters.VersionDir, parameters.LruSize, parameters.GroupInfo) { MarkdownService = markdownService, Processor = processor, Template = templateProcessor, Validators = metadataValidator?.ToImmutableList(), ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor), CanIncrementalBuild = CanProcessorIncremental(processor), }; return(hostService); }
private void Prepare( DocumentBuildParameters parameters, DocumentBuildContext context, TemplateProcessor templateProcessor, string markdownServiceContextHash, out IHostServiceCreator hostServiceCreator, out PhaseProcessor phaseProcessor) { if (IntermediateFolder != null && parameters.ApplyTemplateSettings.TransformDocument) { context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder, markdownServiceContextHash); hostServiceCreator = new HostServiceCreatorWithIncremental(context); phaseProcessor = new PhaseProcessor { Handlers = { new PrebuildBuildPhaseHandler(context).WithIncremental(), new PostbuildPhaseHandler(context, templateProcessor).WithIncremental(), } }; } else { hostServiceCreator = new HostServiceCreator(context); phaseProcessor = new PhaseProcessor { Handlers = { new PrebuildBuildPhaseHandler(context), new PostbuildPhaseHandler(context, templateProcessor), } }; } }
public virtual HostService CreateHostService( DocumentBuildParameters parameters, TemplateProcessor templateProcessor, IMarkdownService markdownService, IEnumerable<IInputMetadataValidator> metadataValidator, IDocumentProcessor processor, IEnumerable<FileAndType> files) { var hostService = new HostService( parameters.Files.DefaultBaseDir, files == null ? new FileModel[0] : from file in files select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model where model != null select model) { MarkdownService = markdownService, Processor = processor, Template = templateProcessor, Validators = metadataValidator?.ToImmutableList(), ShouldTraceIncrementalInfo = ShouldProcessorTraceInfo(processor), CanIncrementalBuild = CanProcessorIncremental(processor), }; return hostService; }
public Manifest Build(DocumentBuildParameters parameters) { if (parameters == null) { throw new ArgumentNullException(nameof(parameters)); } if (parameters.OutputBaseDir == null) { throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir)); } if (parameters.Files == null) { throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files)); } if (parameters.MaxParallelism <= 0) { parameters.MaxParallelism = Environment.ProcessorCount; } if (parameters.Metadata == null) { parameters.Metadata = ImmutableDictionary <string, object> .Empty; } return(BuildCore(parameters)); }
private static string ComputeConfigHash(DocumentBuildParameters parameter) { return(JsonConvert.SerializeObject( parameter, new JsonSerializerSettings { ContractResolver = new IncrementalCheckPropertiesResolver() }).GetMd5String()); }
private IEnumerable <HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable <IDocumentProcessor> processors, TemplateProcessor templateProcessor, IHostServiceCreator creator) { var k = from fileItem in ( from file in parameters.Files.EnumerateFiles() from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> { null } select new { file, p }) group fileItem by fileItem.p; var toHandleItems = k.Where(s => s.Key != null); var notToHandleItems = k.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.file.File); } Logger.LogWarning(sb.ToString()); } foreach (var pair in (from processor in processors join item in toHandleItems on processor equals item.Key into g from item in g.DefaultIfEmpty() select new { processor, item, }).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)) { using (new LoggerPhaseScope(pair.processor.Name, true)) { var hostService = creator.CreateHostService( parameters, templateProcessor, MarkdownService, MetadataValidators, pair.processor, pair.item?.Select(f => f.file)); yield return(hostService); } } }
private List <HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable <IDocumentProcessor> processors, TemplateProcessor templateProcessor, IHostServiceCreator creator) { var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> { null } group file by p).ToList(); var toHandleItems = files.Where(s => s.Key != null); var notToHandleItems = files .Where(s => s.Key == null) .SelectMany(s => s) .Where(s => s.Type != DocumentType.Overwrite && !s.File.EndsWith(".yaml.md", StringComparison.OrdinalIgnoreCase) && !s.File.EndsWith(".yml.md", StringComparison.OrdinalIgnoreCase)) .ToList(); if (notToHandleItems.Count > 0) { Logger.LogWarning( $"Unable to handle following files: {notToHandleItems.Select(s => s.File).ToDelimitedString()}. Do they miss `YamlMime` as the first line of file, e.g.: `### YamlMime:ManagedReference`?", code: WarningCodes.Build.UnknownContentType); } try { return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) join item in toHandleItems.AsParallel() on processor equals item.Key into g from item in g.DefaultIfEmpty() where item != null && item.Any(s => s.Type != DocumentType.Overwrite) // when normal file exists then processing is needed select LoggerPhaseScope.WithScope( processor.Name, LogLevel.Verbose, () => creator.CreateHostService( parameters, templateProcessor, MarkdownService, MetadataValidators, processor, item) )).ToList()); } catch (AggregateException ex) { throw ex.GetBaseException(); } }
private List <IDocumentProcessor> LoadSchemaDrivenDocumentProcessors(DocumentBuildParameters parameter) { using (new LoggerPhaseScope(nameof(LoadSchemaDrivenDocumentProcessors))) { var result = new List <IDocumentProcessor>(); SchemaValidateService.RegisterLicense(parameter.SchemaLicense); using (var resource = parameter?.TemplateManager?.CreateTemplateResource()) { if (resource == null || resource.IsEmpty) { return(result); } var markdigMarkdownService = CreateMarkdigMarkdownService(parameter); foreach (var pair in resource.GetResourceStreams(@"^schemas/.*\.schema\.json")) { var fileName = Path.GetFileName(pair.Key); using (new LoggerFileScope(fileName)) { using (var stream = pair.Value) { using (var sr = new StreamReader(stream)) { DocumentSchema schema; try { schema = DocumentSchema.Load(sr, fileName.Remove(fileName.Length - ".schema.json".Length)); } catch (Exception e) { Logger.LogError(e.Message); throw; } var sdp = new SchemaDrivenDocumentProcessor( schema, new CompositionContainer(CompositionContainer.DefaultContainer), markdigMarkdownService, new FolderRedirectionManager(parameter.OverwriteFragmentsRedirectionRules)); Logger.LogVerbose($"\t{sdp.Name} with build steps ({string.Join(", ", from bs in sdp.BuildSteps orderby bs.BuildOrder select bs.Name)})"); result.Add(sdp); } } } } } if (result.Count > 0) { Logger.LogInfo($"{result.Count} schema driven document processor plug-in(s) loaded."); Processors = Processors.Union(result); } return(result); } }
private void SaveDependency(DocumentBuildContext context, DocumentBuildParameters parameters) { var vbi = _currentBuildInfo.Versions.Find(v => v.VersionName == parameters.VersionName); vbi.Dependency = Path.GetRandomFileName(); using (var writer = File.CreateText( Path.Combine(IntermediateFolder, vbi.Dependency))) { context.DependencyGraph.Save(writer); } }
private List <HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable <IDocumentProcessor> processors, TemplateProcessor templateProcessor, IHostServiceCreator creator) { var files = (from file in parameters.Files.EnumerateFiles().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> { null } group file by p).ToList(); var toHandleItems = files.Where(s => s.Key != null); var notToHandleItems = files.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.File); } Logger.LogWarning(sb.ToString()); } try { return((from processor in processors.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) join item in toHandleItems.AsParallel() on processor equals item.Key into g from item in g.DefaultIfEmpty() select LoggerPhaseScope.WithScope( processor.Name, LogLevel.Verbose, () => creator.CreateHostService( parameters, templateProcessor, MarkdownService, MetadataValidators, processor, item) )).ToList()); } catch (AggregateException ex) { throw new DocfxException(ex.InnerException?.Message, ex); } }
private Dictionary <string, FileAttributeItem> ComputeFileAttributes(DocumentBuildParameters parameters) { return((from f in parameters.Files.EnumerateFiles() let fileKey = ((RelativePath)f.File).GetPathFromWorkingFolder().ToString() select new FileAttributeItem { File = fileKey, LastModifiedTime = File.GetLastWriteTimeUtc(f.FullPath), MD5 = File.ReadAllText(f.FullPath).GetMd5String(), }).ToDictionary(a => a.File)); }
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary <string, string> tokens) { return(MarkdownServiceProvider.CreateMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, TemplateDir = parameters.TemplateDir, Extensions = parameters.MarkdownEngineParameters, Tokens = tokens, })); }
private IEnumerable <HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable <IDocumentProcessor> processors, TemplateProcessor templateProcessor, IMarkdownService markdownService) { var k = from fileItem in ( from file in parameters.Files.EnumerateFiles() from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> { null } select new { file, p }) group fileItem by fileItem.p; var toHandleItems = k.Where(s => s.Key != null); var notToHandleItems = k.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.file.File); } Logger.LogWarning(sb.ToString()); } // todo : revert until PreProcessor ready return(from processor in processors join item in toHandleItems on processor equals item.Key into g from item in g.DefaultIfEmpty().AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) select new HostService( parameters.Files.DefaultBaseDir, item == null ? new FileModel[0] : from file in item select Load(processor, parameters.Metadata, parameters.FileMetadata, file.file) into model where model != null select model) { MarkdownService = markdownService, Processor = processor, Template = templateProcessor, Validators = MetadataValidators.ToImmutableList(), }); }
private static string GetXrefMapFileNameWithGroup(DocumentBuildParameters parameters) { if (!string.IsNullOrEmpty(parameters.GroupInfo?.Name)) { return(parameters.GroupInfo.Name + "." + XRefMapFileName); } if (!string.IsNullOrEmpty(parameters.VersionName)) { return(Uri.EscapeDataString(parameters.VersionName) + "." + XRefMapFileName); } return(XRefMapFileName); }
public static ImmutableList <FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters) { var hostService = new HostService( parameters.Files.DefaultBaseDir, from file in parameters.Files.EnumerateFiles() select Load(processor, parameters.Metadata, parameters.FileMetadata, file) into model where model != null select model); BuildCore(processor, hostService, parameters.MaxParallelism); return(hostService.Models); }
public Manifest Build(DocumentBuildParameters parameters) { using (var builder = new SingleDocumentBuilder { Container = _container, IntermediateFolder = IntermediateFolder, MetadataValidators = MetadataValidators, Processors = Processors }) { return(builder.Build(parameters)); } }
private IEnumerable <IInputMetadataValidator> GetMetadataRules(DocumentBuildParameters parameter) { try { var mvb = MarkdownValidatorBuilder.Create(_container, parameter.Files.DefaultBaseDir, parameter.TemplateDir); return(mvb.GetEnabledMetadataRules().ToList()); } catch (Exception ex) { Logger.LogWarning($"Fail to init markdown style, details:{Environment.NewLine}{ex.ToString()}"); return(Enumerable.Empty <IInputMetadataValidator>()); } }
internal Manifest BuildCore(DocumentBuildParameters parameter, IMarkdownServiceProvider markdownServiceProvider, BuildInfo currentBuildInfo, BuildInfo lastBuildInfo) { using var builder = new SingleDocumentBuilder { CurrentBuildInfo = currentBuildInfo, LastBuildInfo = lastBuildInfo, IntermediateFolder = _intermediateFolder, MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(), Processors = Processors, MarkdownServiceProvider = markdownServiceProvider, }; return(builder.Build(parameter)); }
private MarkdigMarkdownService CreateMarkdigMarkdownService(DocumentBuildParameters parameters) { var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(new DocumentBuildContext(parameters), parameters.MaxParallelism); return(new MarkdigMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, TemplateDir = parameters.TemplateDir, Extensions = parameters.MarkdownEngineParameters, Tokens = templateProcessor?.Tokens?.ToImmutableDictionary(), }, new CompositionContainer(CompositionContainer.DefaultContainer))); }
public void PrepareMetadata(DocumentBuildParameters parameters) { foreach (var postProcessor in _postProcessors) { using (new LoggerPhaseScope(postProcessor.ContractName)) { parameters.Metadata = postProcessor.Processor.PrepareMetadata(parameters.Metadata); if (parameters.Metadata == null) { throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null metadata"); } } } }
private MarkdigMarkdownService CreateMarkdigMarkdownService(DocumentBuildParameters parameters) { var resourceProvider = parameters.TemplateManager?.CreateTemplateResource(); return(new MarkdigMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, TemplateDir = parameters.TemplateDir, Extensions = parameters.MarkdownEngineParameters, Tokens = TemplateProcessorUtility.LoadTokens(resourceProvider)?.ToImmutableDictionary(), }, new CompositionContainer(CompositionContainer.DefaultContainer))); }
private static IEnumerable <InnerBuildContext> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable <IDocumentProcessor> processors, TemplateProcessor templateProcessor, IMarkdownService markdownService) { var k = from fileItem in ( from file in parameters.Files.EnumerateFiles() from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List <IDocumentProcessor> { null } select new { file, p }) group fileItem by fileItem.p; var toHandleItems = k.Where(s => s.Key != null); var notToHandleItems = k.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.file.File); } Logger.LogWarning(sb.ToString()); } return(from item in toHandleItems.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) select new InnerBuildContext( new HostService( parameters.Files.DefaultBaseDir, from file in item select Load(item.Key, parameters.Metadata, parameters.FileMetadata, file.file) into model where model != null select model) { MarkdownService = markdownService, }, item.Key, templateProcessor)); }
private void PrepareMetadata(DocumentBuildParameters parameters) { foreach (var postProcessor in _postProcessors) { using (new LoggerPhaseScope($"Prepare metadata in post processor {postProcessor.ContractName}", false)) using (new PerformanceScope($"Prepare metadata in post processor {postProcessor.ContractName}", LogLevel.Verbose)) { parameters.Metadata = postProcessor.Processor.PrepareMetadata(parameters.Metadata); if (parameters.Metadata == null) { throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null metadata"); } } } }
internal Manifest BuildCore(DocumentBuildParameters parameter) { using (var builder = new SingleDocumentBuilder { Container = _container, CurrentBuildInfo = _currentBuildInfo, LastBuildInfo = _lastBuildInfo, IntermediateFolder = _intermediateFolder, MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(), Processors = Processors, }) { return(builder.Build(parameter)); } }
public override HostService CreateHostService( DocumentBuildParameters parameters, TemplateProcessor templateProcessor, IMarkdownService markdownService, IEnumerable<IInputMetadataValidator> metadataValidator, IDocumentProcessor processor, IEnumerable<FileAndType> files) { if (ShouldProcessorTraceInfo(processor)) { IncrementalContext.CreateProcessorInfo(processor); } var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files); PostCreate(hs, files); return hs; }
/// <summary> /// Export xref map file. /// </summary> private static void ExportXRefMap(DocumentBuildParameters parameters, DocumentBuildContext context) { Logger.LogVerbose("Exporting xref map..."); var xrefMap = new XRefMap(); xrefMap.References = (from xref in context.XRefSpecMap.Values.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) select new XRefSpec(xref) { Href = ((RelativePath)context.FileMap[xref.Href]).RemoveWorkingFolder().ToString() + "#" + XRefDetails.GetHtmlId(xref.Uid), }).ToList(); xrefMap.Sort(); YamlUtility.Serialize( Path.Combine(parameters.OutputBaseDir, XRefMapFileName), xrefMap); Logger.LogInfo("XRef map exported."); }
public override HostService CreateHostService( DocumentBuildParameters parameters, TemplateProcessor templateProcessor, IMarkdownService markdownService, IEnumerable <IInputMetadataValidator> metadataValidator, IDocumentProcessor processor, IEnumerable <FileAndType> files) { if (ShouldProcessorTraceInfo(processor)) { IncrementalContext.CreateProcessorInfo(processor); } var hs = base.CreateHostService(parameters, templateProcessor, markdownService, metadataValidator, processor, files); PostCreate(hs, files); return(hs); }
public static ImmutableList <FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService) { var hostService = new HostService( parameters.Files.DefaultBaseDir, from file in parameters.Files.EnumerateFiles() select Load(processor, parameters.Metadata, parameters.FileMetadata, file, false, null, null, null, null) into model where model != null select model) { Processor = processor, MarkdownService = markdownService, DependencyGraph = new DependencyGraph(), }; BuildCore(hostService, parameters.MaxParallelism); return(hostService.Models); }
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters) { var provider = (IMarkdownServiceProvider)_container.GetExport( typeof(IMarkdownServiceProvider), parameters.MarkdownEngineName); if (provider == null) { Logger.LogError($"Unable to find markdown engine: {parameters.MarkdownEngineName}"); throw new DocfxException($"Unable to find markdown engine: {parameters.MarkdownEngineName}"); } Logger.LogInfo($"Markdown engine is {parameters.MarkdownEngineName}"); return(provider.CreateMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, Extensions = parameters.MarkdownEngineParameters, })); }
private IEnumerable <FileModel> LoadModels(IEnumerable <FileAndType> files, DocumentBuildParameters parameters, IDocumentProcessor processor) { var models = new ConcurrentBag <FileModel>(); if (files == null) { return(models); } files.RunAll(file => { var model = Load(processor, parameters.Metadata, parameters.FileMetadata, file); if (model != null) { models.Add(model); } }, _context.MaxParallelism); return(models); }
public static ImmutableList<FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService) { var hostServiceCreator = new HostServiceCreator(null); var hostService = hostServiceCreator.CreateHostService( parameters, null, markdownService, null, processor, parameters.Files.EnumerateFiles()); var phaseProcessor = new PhaseProcessor { Handlers = { new PrebuildBuildPhaseHandler(null), new PostbuildPhaseHandler(null, null), } }; phaseProcessor.Process(new List<HostService> { hostService }, parameters.MaxParallelism); return hostService.Models; }
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary<string, string> tokens) { return MarkdownServiceProvider.CreateMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, TemplateDir = parameters.TemplateDir, Extensions = parameters.MarkdownEngineParameters, Tokens = tokens, }); }
private Manifest BuildCore(DocumentBuildParameters parameters) { using (new LoggerPhaseScope(PhaseName, true)) { Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}."); Directory.CreateDirectory(parameters.OutputBaseDir); var context = new DocumentBuildContext( Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir), parameters.Files.EnumerateFiles(), parameters.ExternalReferencePackages, parameters.XRefMaps, parameters.MaxParallelism, parameters.Files.DefaultBaseDir, parameters.VersionName, parameters.ApplyTemplateSettings, parameters.RootTocPath); Logger.LogVerbose("Start building document..."); // Start building document... List<HostService> hostServices = null; IHostServiceCreator hostServiceCreator = null; PhaseProcessor phaseProcessor = null; try { using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor) { using (new LoggerPhaseScope("Prepare", true)) { if (MarkdownService == null) { MarkdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary()); } Prepare( parameters, context, templateProcessor, (MarkdownService as IHasIncrementalContext)?.GetIncrementalContextHash(), out hostServiceCreator, out phaseProcessor); } using (new LoggerPhaseScope("Load", true)) { hostServices = GetInnerContexts(parameters, Processors, templateProcessor, hostServiceCreator).ToList(); } BuildCore(phaseProcessor, hostServices, context); return new Manifest { Files = context.ManifestItems.ToList(), Homepages = GetHomepages(context), XRefMap = ExportXRefMap(parameters, context), SourceBasePath = StringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory) }; } } finally { if (hostServices != null) { foreach (var item in hostServices) { Cleanup(item); item.Dispose(); } } } } }
private IEnumerable<IInputMetadataValidator> GetMetadataRules(DocumentBuildParameters parameter) { try { var mvb = MarkdownValidatorBuilder.Create(_container, parameter.Files.DefaultBaseDir, parameter.TemplateDir); return mvb.GetEnabledMetadataRules().ToList(); } catch (Exception ex) { Logger.LogWarning($"Fail to init markdown style, details:{Environment.NewLine}{ex.ToString()}"); return Enumerable.Empty<IInputMetadataValidator>(); } }
internal Manifest BuildCore(DocumentBuildParameters parameter) { using (var builder = new SingleDocumentBuilder { Container = _container, CurrentBuildInfo = _currentBuildInfo, LastBuildInfo = _lastBuildInfo, IntermediateFolder = _intermediateFolder, MetadataValidators = MetadataValidators.Concat(GetMetadataRules(parameter)).ToList(), Processors = Processors, }) { return builder.Build(parameter); } }
public static ImmutableList<FileModel> Build(IDocumentProcessor processor, DocumentBuildParameters parameters, IMarkdownService markdownService) { var hostService = new HostService( parameters.Files.DefaultBaseDir, from file in parameters.Files.EnumerateFiles() select Load(processor, parameters.Metadata, parameters.FileMetadata, file, false, null) into model where model != null select model) { Processor = processor, MarkdownService = markdownService, DependencyGraph = new DependencyGraph(), }; BuildCore(new List<HostService> { hostService }, parameters.MaxParallelism, null, null, null); return hostService.Models; }
private IMarkdownService CreateMarkdownService(DocumentBuildParameters parameters, ImmutableDictionary<string, string> tokens) { var provider = (IMarkdownServiceProvider)Container.GetExport( typeof(IMarkdownServiceProvider), parameters.MarkdownEngineName); if (provider == null) { Logger.LogError($"Unable to find markdown engine: {parameters.MarkdownEngineName}"); throw new DocfxException($"Unable to find markdown engine: {parameters.MarkdownEngineName}"); } Logger.LogInfo($"Markdown engine is {parameters.MarkdownEngineName}"); return provider.CreateMarkdownService( new MarkdownServiceParameters { BasePath = parameters.Files.DefaultBaseDir, TemplateDir = parameters.TemplateDir, Extensions = parameters.MarkdownEngineParameters, Tokens = tokens, }); }
private Manifest BuildCore(DocumentBuildParameters parameters) { using (new LoggerPhaseScope(PhaseName, true)) { Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}."); Directory.CreateDirectory(parameters.OutputBaseDir); var context = new DocumentBuildContext( Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir), parameters.Files.EnumerateFiles(), parameters.ExternalReferencePackages, parameters.XRefMaps, parameters.MaxParallelism, parameters.Files.DefaultBaseDir); if (ShouldTraceIncrementalInfo) { context.IncrementalBuildContext = IncrementalBuildContext.Create(parameters, CurrentBuildInfo, LastBuildInfo, IntermediateFolder); Logger.RegisterListener(context.IncrementalBuildContext.CurrentBuildVersionInfo.BuildMessage.GetListener()); if (context.IncrementalBuildContext.CanVersionIncremental) { context.IncrementalBuildContext.LoadChanges(); Logger.LogVerbose($"Before expanding dependency before build, changes: {JsonUtility.Serialize(context.IncrementalBuildContext.ChangeDict, Formatting.Indented)}"); var dependencyGraph = context.IncrementalBuildContext.LastBuildVersionInfo.Dependency; context.IncrementalBuildContext.ExpandDependency(dependencyGraph, d => dependencyGraph.DependencyTypes[d.Type].Phase == BuildPhase.Build || dependencyGraph.DependencyTypes[d.Type].TriggerBuild); Logger.LogVerbose($"After expanding dependency before build, changes: {JsonUtility.Serialize(context.IncrementalBuildContext.ChangeDict, Formatting.Indented)}"); } } Logger.LogVerbose("Start building document..."); // Start building document... List<HostService> hostServices = null; try { using (var templateProcessor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor) { IMarkdownService markdownService; using (new LoggerPhaseScope("CreateMarkdownService", true)) { markdownService = CreateMarkdownService(parameters, templateProcessor.Tokens.ToImmutableDictionary()); } using (new LoggerPhaseScope("Load", true)) { hostServices = GetInnerContexts(parameters, Processors, templateProcessor, markdownService, context).ToList(); } var manifest = BuildCore(hostServices, context, parameters.VersionName).ToList(); // Use manifest from now on using (new LoggerPhaseScope("UpdateContext", true)) { UpdateContext(context); } // Run getOptions from Template using (new LoggerPhaseScope("FeedOptions", true)) { FeedOptions(manifest, context); } // Template can feed back xref map, actually, the anchor # location can only be determined in template using (new LoggerPhaseScope("FeedXRefMap", true)) { FeedXRefMap(manifest, context); } using (new LoggerPhaseScope("UpdateHref", true)) { UpdateHref(manifest, context); } // Afterwards, m.Item.Model.Content is always IDictionary using (new LoggerPhaseScope("ApplySystemMetadata", true)) { ApplySystemMetadata(manifest, context); } // Register global variables after href are all updated IDictionary<string, object> globalVariables; using (new LoggerPhaseScope("FeedGlobalVariables", true)) { globalVariables = FeedGlobalVariables(templateProcessor.Tokens, manifest, context); } // processor to add global variable to the model foreach (var m in templateProcessor.Process(manifest.Select(s => s.Item).ToList(), context, parameters.ApplyTemplateSettings, globalVariables)) { context.ManifestItems.Add(m); } return new Manifest { Files = context.ManifestItems.ToList(), Homepages = GetHomepages(context), XRefMap = ExportXRefMap(parameters, context), SourceBasePath = TypeForwardedToStringExtension.ToNormalizedPath(EnvironmentContext.BaseDirectory) }; } } finally { if (hostServices != null) { foreach (var item in hostServices) { Cleanup(item); item.Dispose(); } } } } }
private IEnumerable<HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable<IDocumentProcessor> processors, TemplateProcessor templateProcessor, IMarkdownService markdownService, DocumentBuildContext context) { var k = from fileItem in ( from file in parameters.Files.EnumerateFiles() from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null } select new { file, p }) group fileItem by fileItem.p; var toHandleItems = k.Where(s => s.Key != null); var notToHandleItems = k.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.file.File); } Logger.LogWarning(sb.ToString()); } // todo : revert until PreProcessor ready foreach (var pair in (from processor in processors join item in toHandleItems on processor equals item.Key into g from item in g.DefaultIfEmpty() select new { processor, item, }).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)) { var incrementalContext = context.IncrementalBuildContext; var processorSupportIncremental = IsProcessorSupportIncremental(pair.processor); bool processorCanIncremental = processorSupportIncremental; if (processorSupportIncremental) { incrementalContext.CreateProcessorInfo(pair.processor); processorCanIncremental = incrementalContext.CanProcessorIncremental(pair.processor); } var hostService = new HostService( parameters.Files.DefaultBaseDir, pair.item == null ? new FileModel[0] : from file in pair.item select Load(pair.processor, parameters.Metadata, parameters.FileMetadata, file.file, processorCanIncremental, context) into model where model != null select model) { MarkdownService = markdownService, Processor = pair.processor, Template = templateProcessor, Validators = MetadataValidators.ToImmutableList(), ShouldTraceIncrementalInfo = processorSupportIncremental, CanIncrementalBuild = processorCanIncremental, }; if (ShouldTraceIncrementalInfo) { using (new LoggerPhaseScope("ReportModelLoadInfo", true)) { var allFiles = pair.item?.Select(f => f.file.File) ?? new string[0]; var loadedFiles = hostService.Models.Select(m => m.FileAndType.File); incrementalContext.ReportModelLoadInfo(hostService, allFiles.Except(loadedFiles), null); incrementalContext.ReportModelLoadInfo(hostService, loadedFiles, BuildPhase.PreBuild); } } yield return hostService; } }
private void BuildDocument(FileCollection files) { var parameters = new DocumentBuildParameters { Files = files, OutputBaseDir = _outputFolder, ApplyTemplateSettings = _applyTemplateSettings, }; using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null)) { builder.Build(parameters); } }
public void Build(DocumentBuildParameters parameter) { Build(new DocumentBuildParameters[] { parameter }, parameter.OutputBaseDir); }
private void BuildDocument(FileCollection files) { var parameters = new DocumentBuildParameters { Files = files, OutputBaseDir = _outputFolder, ApplyTemplateSettings = _applyTemplateSettings, Metadata = new Dictionary<string, object> { ["meta"] = "Hello world!", }.ToImmutableDictionary() }; using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null)) { builder.Build(parameters); } }
private IEnumerable<HostService> GetInnerContexts( DocumentBuildParameters parameters, IEnumerable<IDocumentProcessor> processors, TemplateProcessor templateProcessor, IHostServiceCreator creator) { var k = from fileItem in ( from file in parameters.Files.EnumerateFiles() from p in (from processor in processors let priority = processor.GetProcessingPriority(file) where priority != ProcessingPriority.NotSupported group processor by priority into ps orderby ps.Key descending select ps.ToList()).FirstOrDefault() ?? new List<IDocumentProcessor> { null } select new { file, p }) group fileItem by fileItem.p; var toHandleItems = k.Where(s => s.Key != null); var notToHandleItems = k.Where(s => s.Key == null); foreach (var item in notToHandleItems) { var sb = new StringBuilder(); sb.AppendLine("Cannot handle following file:"); foreach (var f in item) { sb.Append("\t"); sb.AppendLine(f.file.File); } Logger.LogWarning(sb.ToString()); } foreach (var pair in (from processor in processors join item in toHandleItems on processor equals item.Key into g from item in g.DefaultIfEmpty() select new { processor, item, }).AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)) { var hostService = creator.CreateHostService( parameters, templateProcessor, MarkdownService, MetadataValidators, pair.processor, pair.item?.Select(f => f.file)); yield return hostService; } }
/// <summary> /// Export xref map file. /// </summary> private static string ExportXRefMap(DocumentBuildParameters parameters, DocumentBuildContext context) { Logger.LogVerbose("Exporting xref map..."); var xrefMap = new XRefMap(); xrefMap.References = (from xref in context.XRefSpecMap.Values.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism) select new XRefSpec(xref) { Href = ((RelativePath)context.FileMap[UriUtility.GetNonFragment(xref.Href)]).RemoveWorkingFolder() + UriUtility.GetFragment(xref.Href) }).ToList(); xrefMap.Sort(); string xrefMapFileNameWithVersion = string.IsNullOrEmpty(parameters.VersionName) ? XRefMapFileName : parameters.VersionName + "." + XRefMapFileName; YamlUtility.Serialize( Path.Combine(parameters.OutputBaseDir, xrefMapFileNameWithVersion), xrefMap, YamlMime.XRefMap); Logger.LogInfo("XRef map exported."); return xrefMapFileNameWithVersion; }