public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary <string, object> globals)
{
_context = context ?? throw new ArgumentNullException(nameof(context));
_templateCollection = templateCollection;
_settings = settings;
_globalVariables = globals;
}
public DocumentBuildContext(
string buildOutputFolder,
IEnumerable <FileAndType> allSourceFiles,
ImmutableArray <string> externalReferencePackages,
ImmutableArray <string> xrefMaps,
int maxParallelism,
string baseFolder,
string versionName,
ApplyTemplateSettings applyTemplateSetting,
string rootTocPath)
{
BuildOutputFolder = buildOutputFolder;
VersionName = versionName;
ApplyTemplateSettings = applyTemplateSetting;
AllSourceFiles = GetAllSourceFiles(allSourceFiles);
ExternalReferencePackages = externalReferencePackages;
XRefMapUrls = xrefMaps;
MaxParallelism = maxParallelism;
if (xrefMaps.Length > 0)
{
_reader = new XRefCollection(
from u in xrefMaps
select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder);
}
RootTocPath = rootTocPath;
}
internal List<ManifestItem> Process(List<InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals = null)
{
using (new LoggerPhaseScope("Apply Templates", true))
{
if (globals == null)
{
globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
}
var documentTypes = manifest.Select(s => s.DocumentType).Distinct();
var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null);
if (notSupportedDocumentTypes.Any())
{
Logger.LogWarning($"There is no template processing document type(s): {TypeForwardedToStringExtension.ToDelimitedString(notSupportedDocumentTypes)}");
}
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList();
ProcessDependencies(settings.OutputFolder, templatesInUse);
}
else
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
var templateManifest = ProcessCore(manifest, context, settings, globals);
return templateManifest;
}
}
public DocumentBuildContext(
string buildOutputFolder,
IEnumerable<FileAndType> allSourceFiles,
ImmutableArray<string> externalReferencePackages,
ImmutableArray<string> xrefMaps,
int maxParallelism,
string baseFolder,
string versionName,
ApplyTemplateSettings applyTemplateSetting,
string rootTocPath)
{
BuildOutputFolder = buildOutputFolder;
VersionName = versionName;
ApplyTemplateSettings = applyTemplateSetting;
AllSourceFiles = GetAllSourceFiles(allSourceFiles);
ExternalReferencePackages = externalReferencePackages;
XRefMapUrls = xrefMaps;
MaxParallelism = maxParallelism;
if (xrefMaps.Length > 0)
{
_reader = new XRefCollection(
from u in xrefMaps
select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder);
}
RootTocPath = rootTocPath;
}
public TocDocumentProcessorTest()
{
_outputFolder = GetRandomFolder();
_inputFolder = GetRandomFolder();
_templateFolder = GetRandomFolder();
_applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
_applyTemplateSettings.RawModelExportSettings.Export = true;
_fileCreator = new FileCreator(_inputFolder);
}
public RestApiDocumentProcessorTest()
{
_outputFolder = GetRandomFolder();
_inputFolder = GetRandomFolder();
_templateFolder = GetRandomFolder();
_defaultFiles = new FileCollection(Directory.GetCurrentDirectory());
_defaultFiles.Add(DocumentType.Article, new[] { "TestData/contacts.json" }, "TestData/");
_applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
_applyTemplateSettings.RawModelExportSettings.Export = true;
}
private static void ExportRawModel(List <ManifestItem> manifest, ApplyTemplateSettings settings)
{
if (!settings.Options.HasFlag(ApplyTemplateOptions.ExportRawModel))
{
return;
}
Logger.LogInfo($"Exporting {manifest.Count} raw model(s)...");
foreach (var item in manifest)
{
ExportModel(item.Model.Content, item.ModelFile, settings.RawModelExportSettings);
}
}
public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary<string, object> globals)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_context = context;
_templateCollection = templateCollection;
_settings = settings;
_globalVariables = globals;
_systemMetadataGenerator = new SystemMetadataGenerator(context);
}
public TemplateModelTransformer(DocumentBuildContext context, TemplateCollection templateCollection, ApplyTemplateSettings settings, IDictionary <string, object> globals)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_context = context;
_templateCollection = templateCollection;
_settings = settings;
_globalVariables = globals;
_systemMetadataGenerator = new SystemMetadataGenerator(context);
}
public DocumentBuildContext(
string buildOutputFolder,
IEnumerable <FileAndType> allSourceFiles,
ImmutableArray <string> externalReferencePackages,
ImmutableArray <string> xrefMaps,
int maxParallelism,
string baseFolder,
string versionName,
ApplyTemplateSettings applyTemplateSetting,
string rootTocPath,
string versionFolder,
ImmutableArray <string> xrefServiceUrls,
GroupInfo groupInfo,
List <string> xrefTags)
{
BuildOutputFolder = buildOutputFolder;
VersionName = versionName;
ApplyTemplateSettings = applyTemplateSetting;
HrefGenerator = applyTemplateSetting?.HrefGenerator;
AllSourceFiles = GetAllSourceFiles(allSourceFiles);
ExternalReferencePackages = externalReferencePackages;
_xrefMapUrls = xrefMaps;
_xrefServiceUrls = xrefServiceUrls;
GroupInfo = groupInfo;
XRefTags = xrefTags;
MaxParallelism = maxParallelism;
MaxHttpParallelism = maxParallelism * 2;
if (xrefMaps.Length > 0)
{
_reader = new XRefCollection(
from u in xrefMaps
select new Uri(u, UriKind.RelativeOrAbsolute)).GetReaderAsync(baseFolder);
}
RootTocPath = rootTocPath;
if (!string.IsNullOrEmpty(versionFolder) && Path.IsPathRooted(versionFolder))
{
throw new ArgumentException("Path cannot be rooted.", nameof(versionFolder));
}
if (!string.IsNullOrEmpty(versionFolder))
{
versionFolder = versionFolder.Replace('\\', '/');
if (!versionFolder.EndsWith("/", StringComparison.Ordinal))
{
versionFolder += "/";
}
}
VersionFolder = versionFolder;
}
private List <ManifestItem> ProcessCore(List <InternalManifestItem> items, ApplyTemplateSettings settings, IDictionary <string, object> globals)
{
var manifest = new ConcurrentBag <ManifestItem>();
var transformer = new TemplateModelTransformer(_context, _templateCollection, settings, globals);
items.RunAll(
item =>
{
using (new LoggerFileScope(item.LocalPathFromRoot))
{
manifest.Add(transformer.Transform(item));
}
},
_maxParallelism);
return(manifest.ToList());
}
internal void ProcessDependencies(HashSet<string> documentTypes, ApplyTemplateSettings settings)
{
if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null);
if (notSupportedDocumentTypes.Any())
{
Logger.LogWarning($"There is no template processing document type(s): {StringExtension.ToDelimitedString(notSupportedDocumentTypes)}");
}
var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList();
ProcessDependenciesCore(settings.OutputFolder, templatesInUse);
}
else
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
}
internal void ProcessDependencies(HashSet <string> documentTypes, ApplyTemplateSettings settings)
{
if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null).OrderBy(s => s);
if (notSupportedDocumentTypes.Any())
{
Logger.LogWarning($"There is no template processing document type(s): {StringExtension.ToDelimitedString(notSupportedDocumentTypes)}");
}
var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList();
ProcessDependenciesCore(settings.OutputFolder, templatesInUse);
}
else
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
}
internal List<ManifestItem> Process(List<InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals = null)
{
using (new LoggerPhaseScope("Apply Templates", true))
{
if (globals == null)
{
globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
}
if (settings == null)
{
settings = context.ApplyTemplateSettings;
}
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
var documentTypes = new HashSet<string>(manifest.Select(s => s.DocumentType));
ProcessDependencies(documentTypes, settings);
var templateManifest = ProcessCore(manifest, context, settings, globals);
return templateManifest;
}
}
internal List <ManifestItem> Process(List <InternalManifestItem> manifest, ApplyTemplateSettings settings, IDictionary <string, object> globals = null)
{
using (new LoggerPhaseScope("Apply Templates", LogLevel.Verbose))
{
if (globals == null)
{
globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
}
if (settings == null)
{
settings = _context?.ApplyTemplateSettings;
}
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
var documentTypes = new HashSet <string>(manifest.Select(s => s.DocumentType));
ProcessDependencies(documentTypes, settings);
var templateManifest = ProcessCore(manifest, settings, globals);
return(templateManifest);
}
}
public List <TemplateManifestItem> Transform(List <ManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings)
{
var documentTypes = items.Select(s => s.DocumentType).Distinct().Where(s => s != "Resource" && Templates[s] == null);
if (documentTypes.Any())
{
Logger.LogWarning($"There is no template processing document type(s): {documentTypes.ToDelimitedString()}");
}
var manifest = new ConcurrentBag <TemplateManifestItem>();
var systemAttributeGenerator = new SystemMetadataGenerator(context);
items.RunAll(
item =>
{
var manifestItem = TransformItem(item, context, settings, systemAttributeGenerator);
if (manifestItem != null)
{
manifest.Add(manifestItem);
}
},
context.MaxParallelism); // todo : set parallelism.
return(manifest.ToList());
}
private TemplateManifestItem TransformItem(ManifestItem item, IDocumentBuildContext context, ApplyTemplateSettings settings, SystemMetadataGenerator systemAttributeGenerator)
{
if (settings.Options.HasFlag(ApplyTemplateOptions.ExportRawModel))
{
ExportModel(item.Model.Content, item.ModelFile, settings.RawModelExportSettings);
}
if (item.Model == null || item.Model.Content == null)
{
throw new ArgumentNullException("Content for item.Model should not be null!");
}
var manifestItem = new TemplateManifestItem
{
DocumentType = item.DocumentType,
OriginalFile = item.LocalPathFromRepoRoot,
OutputFiles = new Dictionary <string, string>()
};
var outputDirectory = settings.OutputFolder ?? Environment.CurrentDirectory;
if (!IsEmpty)
{
HashSet <string> missingUids = new HashSet <string>();
// Must convert to JObject first as we leverage JsonProperty as the property name for the model
var model = ConvertToObjectHelper.ConvertStrongTypeToJObject(item.Model.Content);
var templates = Templates[item.DocumentType];
// 1. process model
if (templates != null)
{
var systemAttrs = systemAttributeGenerator.Generate(item);
foreach (var template in templates)
{
var extension = template.Extension;
string outputFile = Path.ChangeExtension(item.ModelFile, extension);
string outputPath = Path.Combine(outputDirectory, outputFile);
var dir = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(dir))
{
Directory.CreateDirectory(dir);
}
object viewModel = null;
try
{
viewModel = template.TransformModel(model, systemAttrs, _global);
}
catch (Exception e)
{
// save raw model for further investigation:
var exportSettings = ApplyTemplateSettings.RawModelExportSettingsForDebug;
var rawModelPath = ExportModel(model, item.ModelFile, exportSettings);
throw new DocumentException($"Error transforming model \"{rawModelPath}\" generated from \"{item.LocalPathFromRepoRoot}\" using \"{template.ScriptName}\": {e.Message}");
}
string result;
try
{
result = template.Transform(viewModel);
}
catch (Exception e)
{
// save view model for further investigation:
var exportSettings = ApplyTemplateSettings.ViewModelExportSettingsForDebug;
var viewModelPath = ExportModel(viewModel, outputFile, exportSettings);
throw new DocumentException($"Error applying template \"{template.Name}\" to view model \"{viewModelPath}\" generated from \"{item.LocalPathFromRepoRoot}\": {e.Message}");
}
if (settings.Options.HasFlag(ApplyTemplateOptions.ExportViewModel))
{
ExportModel(viewModel, outputFile, settings.ViewModelExportSettings);
}
if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
if (string.IsNullOrWhiteSpace(result))
{
// TODO: WHAT to do if is transformed to empty string? STILL creat empty file?
Logger.LogWarning($"Model \"{item.ModelFile}\" is transformed to empty string with template \"{template.Name}\"");
File.WriteAllText(outputPath, string.Empty);
}
else
{
TransformDocument(result, extension, context, outputPath, item.ModelFile, missingUids);
Logger.Log(LogLevel.Verbose, $"Transformed model \"{item.LocalPathFromRepoRoot}\" to \"{outputPath}\".");
}
manifestItem.OutputFiles.Add(extension, outputFile);
}
}
}
if (missingUids.Count > 0)
{
var uids = string.Join(", ", missingUids.Select(s => $"\"{s}\""));
Logger.LogWarning($"Unable to resolve cross-reference {uids} for \"{manifestItem.OriginalFile.ToDisplayPath()}\"");
}
}
// 2. process resource
if (item.ResourceFile != null)
{
// Resource file has already been processed in its plugin
manifestItem.OutputFiles.Add("resource", item.ResourceFile);
}
return(manifestItem);
}
public DocumentBuildContext(string buildOutputFolder, IEnumerable <FileAndType> allSourceFiles, ImmutableArray <string> externalReferencePackages, ImmutableArray <string> xrefMaps, int maxParallelism, string baseFolder, string versionName, ApplyTemplateSettings applyTemplateSetting, string rootTocPath, string versionFolder, ImmutableArray <string> xrefServiceUrls)
: this(buildOutputFolder, allSourceFiles, externalReferencePackages, xrefMaps, maxParallelism, baseFolder, versionName, applyTemplateSetting, rootTocPath, null, ImmutableArray <string> .Empty, null, null)
{
}
public static List <TemplateManifestItem> Transform(TemplateProcessor processor, List <ManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings)
{
if (settings.Options == ApplyTemplateOptions.ExportRawModel || processor == null)
{
ExportRawModel(manifest, settings);
return(null);
}
using (new LoggerPhaseScope("Apply Templates"))
{
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
processor.ProcessDependencies(settings.OutputFolder);
if (processor.IsEmpty)
{
Logger.LogWarning("No template is found.");
ExportRawModel(manifest, settings);
return(null);
}
Logger.LogVerbose("Start applying template...");
var outputDirectory = context.BuildOutputFolder;
var templateManifest = processor.Transform(manifest, context, settings);
if (!settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
if (templateManifest.Count > 0)
{
// Save manifest from template
// TODO: Keep .manifest for backward-compatability, will remove next sprint
var manifestPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ObsoleteManifestFileName);
JsonUtility.Serialize(manifestPath, templateManifest);
// Logger.LogInfo($"Manifest file saved to {manifestPath}. NOTE: This file is out-of-date and will be removed in version 1.8, if you rely on this file, please change to use {Constants.ManifestFileName} instead.");
var manifestJsonPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ManifestFileName);
var toc = context.GetTocInfo();
var manifestObject = GenerateManifest(context, templateManifest);
JsonUtility.Serialize(manifestJsonPath, manifestObject);
Logger.LogInfo($"Manifest file saved to {manifestJsonPath}.");
}
return(templateManifest);
}
}
private List <TemplateManifestItem> ProcessCore(List <ManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals)
{
var manifest = new ConcurrentBag <TemplateManifestItem>();
var systemAttributeGenerator = new SystemMetadataGenerator(context);
var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals);
items.RunAll(
item =>
{
var manifestItem = transformer.Transform(item);
if (manifestItem.OutputFiles?.Count > 0)
{
manifest.Add(manifestItem);
}
},
context.MaxParallelism);
var itemsToRemove = new List <string>();
foreach (var duplicates in from m in manifest
from output in m.OutputFiles.Values
group m.OriginalFile by output into g
where g.Count() > 1
select g)
{
Logger.LogWarning($"Overwrite occurs while input files \"{string.Join(", ", duplicates)}\" writing to the same output file \"{duplicates.Key}\"");
itemsToRemove.AddRange(duplicates.Skip(1));
}
return(manifest.Where(m => !itemsToRemove.Contains(m.OriginalFile)).ToList());
}
private List<ManifestItem> ProcessCore(List<InternalManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary<string, object> globals)
{
var manifest = new ConcurrentBag<ManifestItem>();
var systemAttributeGenerator = new SystemMetadataGenerator(context);
var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals);
items.RunAll(
item =>
{
using (new LoggerFileScope(item.LocalPathFromRoot))
{
var manifestItem = transformer.Transform(item);
if (manifestItem.OutputFiles?.Count > 0)
{
manifest.Add(manifestItem);
}
}
},
context.MaxParallelism);
return manifest.ToList();
}
private List <ManifestItem> ProcessCore(List <InternalManifestItem> items, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals)
{
var manifest = new ConcurrentBag <ManifestItem>();
var systemAttributeGenerator = new SystemMetadataGenerator(context);
var transformer = new TemplateModelTransformer(context, _templateCollection, settings, globals);
items.RunAll(
item =>
{
using (new LoggerFileScope(item.LocalPathFromRoot))
{
var manifestItem = transformer.Transform(item);
if (manifestItem.OutputFiles?.Count > 0)
{
manifest.Add(manifestItem);
}
}
},
context.MaxParallelism);
return(manifest.ToList());
}
internal List <ManifestItem> Process(List <InternalManifestItem> manifest, DocumentBuildContext context, ApplyTemplateSettings settings, IDictionary <string, object> globals = null)
{
using (new LoggerPhaseScope("Apply Templates", true))
{
if (globals == null)
{
globals = Tokens.ToDictionary(pair => pair.Key, pair => (object)pair.Value);
}
var documentTypes = manifest.Select(s => s.DocumentType).Distinct();
var notSupportedDocumentTypes = documentTypes.Where(s => s != "Resource" && _templateCollection[s] == null);
if (notSupportedDocumentTypes.Any())
{
Logger.LogWarning($"There is no template processing document type(s): {notSupportedDocumentTypes.ToDelimitedString()}");
}
Logger.LogInfo($"Applying templates to {manifest.Count} model(s)...");
if (settings.Options.HasFlag(ApplyTemplateOptions.TransformDocument))
{
var templatesInUse = documentTypes.Select(s => _templateCollection[s]).Where(s => s != null).ToList();
ProcessDependencies(settings.OutputFolder, templatesInUse);
}
else
{
Logger.LogInfo("Dryrun, no template will be applied to the documents.");
}
var templateManifest = ProcessCore(manifest, context, settings, globals);
return(templateManifest);
}
}
