public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary <string, object> globals) { _context = context ?? throw new ArgumentNullException(nameof(context)); _templateCollection = templateCollection; _settings = settings; _globalVariables = globals; }
public DocumentBuildContext( string buildOutputFolder, IEnumerable <FileAndType> allSourceFiles, ImmutableArray <string> externalReferencePackages, ImmutableArray <string> xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath) { BuildOutputFolder = buildOutputFolder; VersionName = versionName; ApplyTemplateSettings = applyTemplateSetting; AllSourceFiles = GetAllSourceFiles(allSourceFiles); ExternalReferencePackages = externalReferencePackages; XRefMapUrls = xrefMaps; MaxParallelism = maxParallelism; if (xrefMaps.Length > 0) { _reader = new XRefCollection( from u in xrefMaps select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder); } RootTocPath = rootTocPath; }
internal List<ManifestItem> Process(List<InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals = null) { using (new LoggerPhaseScope("Apply Templates", true)) { if (globals == null) { globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value); } var documentTypes = manifest.Select(s => s.DocumentType).Distinct(); var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null); if (notSupportedDocumentTypes.Any()) { Logger.LogWarning($"There is no template processing document type(s): {TypeForwardedToStringExtension.ToDelimitedString(notSupportedDocumentTypes)}"); } Logger.LogInfo($"Applying templates to {manifest.Count} model(s)..."); if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList(); ProcessDependencies(settings.OutputFolder, templatesInUse); } else { Logger.LogInfo("Dryrun, no template will be applied to the documents."); } var templateManifest = ProcessCore(manifest, context, settings, globals); return templateManifest; } }
public DocumentBuildContext( string buildOutputFolder, IEnumerable<FileAndType> allSourceFiles, ImmutableArray<string> externalReferencePackages, ImmutableArray<string> xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath) { BuildOutputFolder = buildOutputFolder; VersionName = versionName; ApplyTemplateSettings = applyTemplateSetting; AllSourceFiles = GetAllSourceFiles(allSourceFiles); ExternalReferencePackages = externalReferencePackages; XRefMapUrls = xrefMaps; MaxParallelism = maxParallelism; if (xrefMaps.Length > 0) { _reader = new XRefCollection( from u in xrefMaps select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder); } RootTocPath = rootTocPath; }
public TocDocumentProcessorTest() { _outputFolder = GetRandomFolder(); _inputFolder = GetRandomFolder(); _templateFolder = GetRandomFolder(); _applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder); _applyTemplateSettings.RawModelExportSettings.Export = true; _fileCreator = new FileCreator(_inputFolder); }
public RestApiDocumentProcessorTest() { _outputFolder = GetRandomFolder(); _inputFolder = GetRandomFolder(); _templateFolder = GetRandomFolder(); _defaultFiles = new FileCollection(Directory.GetCurrentDirectory()); _defaultFiles.Add(DocumentType.Article, new[] { "TestData/contacts.json" }, "TestData/"); _applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder); _applyTemplateSettings.RawModelExportSettings.Export = true; }
private static void ExportRawModel(List <ManifestItem> manifest, ApplyTemplateSettings settings) { if (!settings.Options.HasFlag(ApplyTemplateOptions.ExportRawModel)) { return; } Logger.LogInfo($"Exporting {manifest.Count} raw model(s)..."); foreach (var item in manifest) { ExportModel(item.Model.Content, item.ModelFile, settings.RawModelExportSettings); } }
public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary<string, object> globals) { if (context == null) { throw new ArgumentNullException(nameof(context)); } _context = context; _templateCollection = templateCollection; _settings = settings; _globalVariables = globals; _systemMetadataGenerator = new SystemMetadataGenerator(context); }
public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary <string, object> globals) { if (context == null) { throw new ArgumentNullException(nameof(context)); } _context = context; _templateCollection = templateCollection; _settings = settings; _globalVariables = globals; _systemMetadataGenerator = new SystemMetadataGenerator(context); }
public DocumentBuildContext( string buildOutputFolder, IEnumerable <FileAndType> allSourceFiles, ImmutableArray <string> externalReferencePackages, ImmutableArray <string> xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder, ImmutableArray <string> xrefServiceUrls, GroupInfo groupInfo, List <string> xrefTags) { BuildOutputFolder = buildOutputFolder; VersionName = versionName; ApplyTemplateSettings = applyTemplateSetting; HrefGenerator = applyTemplateSetting?.HrefGenerator; AllSourceFiles = GetAllSourceFiles(allSourceFiles); ExternalReferencePackages = externalReferencePackages; _xrefMapUrls = xrefMaps; _xrefServiceUrls = xrefServiceUrls; GroupInfo = groupInfo; XRefTags = xrefTags; MaxParallelism = maxParallelism; MaxHttpParallelism = maxParallelism * 2; if (xrefMaps.Length > 0) { _reader = new XRefCollection( from u in xrefMaps select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder); } RootTocPath = rootTocPath; if (!string.IsNullOrEmpty(versionFolder) && Path.IsPathRooted(versionFolder)) { throw new ArgumentException("Path cannot be rooted.", nameof(versionFolder)); } if (!string.IsNullOrEmpty(versionFolder)) { versionFolder = versionFolder.Replace('\\', '/'); if (!versionFolder.EndsWith("/", StringComparison.Ordinal)) { versionFolder += "/"; } } VersionFolder = versionFolder; }
private List <ManifestItem> ProcessCore(List <InternalManifestItem> items, ApplyTemplateSettings settings, IDictionary <string, object> globals) { var manifest = new ConcurrentBag <ManifestItem>(); var transformer = new TemplateModelTransformer(_context, _templateCollection, settings, globals); items.RunAll( item => { using (new LoggerFileScope(item.LocalPathFromRoot)) { manifest.Add(transformer.Transform(item)); } }, _maxParallelism); return(manifest.ToList()); }
internal void ProcessDependencies(HashSet<string> documentTypes, ApplyTemplateSettings settings) { if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null); if (notSupportedDocumentTypes.Any()) { Logger.LogWarning($"There is no template processing document type(s): {StringExtension.ToDelimitedString(notSupportedDocumentTypes)}"); } var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList(); ProcessDependenciesCore(settings.OutputFolder, templatesInUse); } else { Logger.LogInfo("Dryrun, no template will be applied to the documents."); } }
internal void ProcessDependencies(HashSet <string> documentTypes, ApplyTemplateSettings settings) { if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null).OrderBy(s => s); if (notSupportedDocumentTypes.Any()) { Logger.LogWarning($"There is no template processing document type(s): {StringExtension.ToDelimitedString(notSupportedDocumentTypes)}"); } var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList(); ProcessDependenciesCore(settings.OutputFolder, templatesInUse); } else { Logger.LogInfo("Dryrun, no template will be applied to the documents."); } }
internal List<ManifestItem> Process(List<InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals = null) { using (new LoggerPhaseScope("Apply Templates", true)) { if (globals == null) { globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value); } if (settings == null) { settings = context.ApplyTemplateSettings; } Logger.LogInfo($"Applying templates to {manifest.Count} model(s)..."); var documentTypes = new HashSet<string>(manifest.Select(s => s.DocumentType)); ProcessDependencies(documentTypes, settings); var templateManifest = ProcessCore(manifest, context, settings, globals); return templateManifest; } }
internal List <ManifestItem> Process(List <InternalManifestItem> manifest, ApplyTemplateSettings settings, IDictionary <string, object> globals = null) { using (new LoggerPhaseScope("Apply Templates", LogLevel.Verbose)) { if (globals == null) { globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value); } if (settings == null) { settings = _context?.ApplyTemplateSettings; } Logger.LogInfo($"Applying templates to {manifest.Count} model(s)..."); var documentTypes = new HashSet <string>(manifest.Select(s => s.DocumentType)); ProcessDependencies(documentTypes, settings); var templateManifest = ProcessCore(manifest, settings, globals); return(templateManifest); } }
public List <TemplateManifestItem> Transform(List <ManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings) { var documentTypes = items.Select(s => s.DocumentType).Distinct().Where(s => s != "Resource" && Templates[s] == null); if (documentTypes.Any()) { Logger.LogWarning($"There is no template processing document type(s): {documentTypes.ToDelimitedString()}"); } var manifest = new ConcurrentBag <TemplateManifestItem>(); var systemAttributeGenerator = new SystemMetadataGenerator(context); items.RunAll( item => { var manifestItem = TransformItem(item, context, settings, systemAttributeGenerator); if (manifestItem != null) { manifest.Add(manifestItem); } }, context.MaxParallelism); // todo : set parallelism. return(manifest.ToList()); }
private TemplateManifestItem TransformItem(ManifestItem item, IDocumentBuildContext context, ApplyTemplateSettings settings, SystemMetadataGenerator systemAttributeGenerator) { if (settings.Options.HasFlag(ApplyTemplateOptions.ExportRawModel)) { ExportModel(item.Model.Content, item.ModelFile, settings.RawModelExportSettings); } if (item.Model == null || item.Model.Content == null) { throw new ArgumentNullException("Content for item.Model should not be null!"); } var manifestItem = new TemplateManifestItem { DocumentType = item.DocumentType, OriginalFile = item.LocalPathFromRepoRoot, OutputFiles = new Dictionary <string, string>() }; var outputDirectory = settings.OutputFolder ?? Environment.CurrentDirectory; if (!IsEmpty) { HashSet <string> missingUids = new HashSet <string>(); // Must convert to JObject first as we leverage JsonProperty as the property name for the model var model = ConvertToObjectHelper.ConvertStrongTypeToJObject(item.Model.Content); var templates = Templates[item.DocumentType]; // 1. process model if (templates != null) { var systemAttrs = systemAttributeGenerator.Generate(item); foreach (var template in templates) { var extension = template.Extension; string outputFile = Path.ChangeExtension(item.ModelFile, extension); string outputPath = Path.Combine(outputDirectory, outputFile); var dir = Path.GetDirectoryName(outputPath); if (!string.IsNullOrEmpty(dir)) { Directory.CreateDirectory(dir); } object viewModel = null; try { viewModel = template.TransformModel(model, systemAttrs, _global); } catch (Exception e) { // save raw model for further investigation: var exportSettings = ApplyTemplateSettings.RawModelExportSettingsForDebug; var rawModelPath = ExportModel(model, item.ModelFile, exportSettings); throw new DocumentException($"Error transforming model \"{rawModelPath}\" generated from \"{item.LocalPathFromRepoRoot}\" using \"{template.ScriptName}\": {e.Message}"); } string result; try { result = template.Transform(viewModel); } catch (Exception e) { // save view model for further investigation: var exportSettings = ApplyTemplateSettings.ViewModelExportSettingsForDebug; var viewModelPath = ExportModel(viewModel, outputFile, exportSettings); throw new DocumentException($"Error applying template \"{template.Name}\" to view model \"{viewModelPath}\" generated from \"{item.LocalPathFromRepoRoot}\": {e.Message}"); } if (settings.Options.HasFlag(ApplyTemplateOptions.ExportViewModel)) { ExportModel(viewModel, outputFile, settings.ViewModelExportSettings); } if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { if (string.IsNullOrWhiteSpace(result)) { // TODO: WHAT to do if is transformed to empty string? STILL creat empty file? Logger.LogWarning($"Model \"{item.ModelFile}\" is transformed to empty string with template \"{template.Name}\""); File.WriteAllText(outputPath, string.Empty); } else { TransformDocument(result, extension, context, outputPath, item.ModelFile, missingUids); Logger.Log(LogLevel.Verbose, $"Transformed model \"{item.LocalPathFromRepoRoot}\" to \"{outputPath}\"."); } manifestItem.OutputFiles.Add(extension, outputFile); } } } if (missingUids.Count > 0) { var uids = string.Join(", ", missingUids.Select(s => $"\"{s}\"")); Logger.LogWarning($"Unable to resolve cross-reference {uids} for \"{manifestItem.OriginalFile.ToDisplayPath()}\""); } } // 2. process resource if (item.ResourceFile != null) { // Resource file has already been processed in its plugin manifestItem.OutputFiles.Add("resource", item.ResourceFile); } return(manifestItem); }
public DocumentBuildContext(string buildOutputFolder, IEnumerable <FileAndType> allSourceFiles, ImmutableArray <string> externalReferencePackages, ImmutableArray <string> xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder, ImmutableArray <string> xrefServiceUrls) : this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null, ImmutableArray <string> .Empty, null, null) { }
public static List <TemplateManifestItem> Transform(TemplateProcessor processor, List <ManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings) { if (settings.Options == ApplyTemplateOptions.ExportRawModel || processor == null) { ExportRawModel(manifest, settings); return(null); } using (new LoggerPhaseScope("Apply Templates")) { Logger.LogInfo($"Applying templates to {manifest.Count} model(s)..."); processor.ProcessDependencies(settings.OutputFolder); if (processor.IsEmpty) { Logger.LogWarning("No template is found."); ExportRawModel(manifest, settings); return(null); } Logger.LogVerbose("Start applying template..."); var outputDirectory = context.BuildOutputFolder; var templateManifest = processor.Transform(manifest, context, settings); if (!settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { Logger.LogInfo("Dryrun, no template will be applied to the documents."); } if (templateManifest.Count > 0) { // Save manifest from template // TODO: Keep .manifest for backward-compatability, will remove next sprint var manifestPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ObsoleteManifestFileName); JsonUtility.Serialize(manifestPath, templateManifest); // Logger.LogInfo($"Manifest file saved to {manifestPath}. NOTE: This file is out-of-date and will be removed in version 1.8, if you rely on this file, please change to use {Constants.ManifestFileName} instead."); var manifestJsonPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ManifestFileName); var toc = context.GetTocInfo(); var manifestObject = GenerateManifest(context, templateManifest); JsonUtility.Serialize(manifestJsonPath, manifestObject); Logger.LogInfo($"Manifest file saved to {manifestJsonPath}."); } return(templateManifest); } }
private List <TemplateManifestItem> ProcessCore(List <ManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals) { var manifest = new ConcurrentBag <TemplateManifestItem>(); var systemAttributeGenerator = new SystemMetadataGenerator(context); var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals); items.RunAll( item => { var manifestItem = transformer.Transform(item); if (manifestItem.OutputFiles?.Count > 0) { manifest.Add(manifestItem); } }, context.MaxParallelism); var itemsToRemove = new List <string>(); foreach (var duplicates in from m in manifest from output in m.OutputFiles.Values group m.OriginalFile by output into g where g.Count() > 1 select g) { Logger.LogWarning($"Overwrite occurs while input files \"{string.Join(", ", duplicates)}\" writing to the same output file \"{duplicates.Key}\""); itemsToRemove.AddRange(duplicates.Skip(1)); } return(manifest.Where(m => !itemsToRemove.Contains(m.OriginalFile)).ToList()); }
private List<ManifestItem> ProcessCore(List<InternalManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals) { var manifest = new ConcurrentBag<ManifestItem>(); var systemAttributeGenerator = new SystemMetadataGenerator(context); var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals); items.RunAll( item => { using (new LoggerFileScope(item.LocalPathFromRoot)) { var manifestItem = transformer.Transform(item); if (manifestItem.OutputFiles?.Count > 0) { manifest.Add(manifestItem); } } }, context.MaxParallelism); return manifest.ToList(); }
private List <ManifestItem> ProcessCore(List <InternalManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals) { var manifest = new ConcurrentBag <ManifestItem>(); var systemAttributeGenerator = new SystemMetadataGenerator(context); var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals); items.RunAll( item => { using (new LoggerFileScope(item.LocalPathFromRoot)) { var manifestItem = transformer.Transform(item); if (manifestItem.OutputFiles?.Count > 0) { manifest.Add(manifestItem); } } }, context.MaxParallelism); return(manifest.ToList()); }
internal List <ManifestItem> Process(List <InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals = null) { using (new LoggerPhaseScope("Apply Templates", true)) { if (globals == null) { globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value); } var documentTypes = manifest.Select(s => s.DocumentType).Distinct(); var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null); if (notSupportedDocumentTypes.Any()) { Logger.LogWarning($"There is no template processing document type(s): {notSupportedDocumentTypes.ToDelimitedString()}"); } Logger.LogInfo($"Applying templates to {manifest.Count} model(s)..."); if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument)) { var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList(); ProcessDependencies(settings.OutputFolder, templatesInUse); } else { Logger.LogInfo("Dryrun, no template will be applied to the documents."); } var templateManifest = ProcessCore(manifest, context, settings, globals); return(templateManifest); } }