public static IncrementalBuildContext Create(DocumentBuildParameters parameters, BuildInfo cb, BuildInfo lb, string intermediateFolder)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (cb == null)
{
throw new ArgumentNullException(nameof(cb));
}
if (intermediateFolder == null)
{
throw new ArgumentNullException(nameof(intermediateFolder));
}
var baseDir = Path.Combine(intermediateFolder, cb.DirectoryName);
var lastBaseDir = lb != null ? Path.Combine(intermediateFolder, lb.DirectoryName) : null;
var lastBuildStartTime = lb?.BuildStartTime;
var canBuildInfoIncremental = CanBuildInfoIncremental(cb, lb);
var lbv = lb?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
var cbv = new BuildVersionInfo
{
VersionName = parameters.VersionName,
ConfigHash = ComputeConfigHash(parameters),
AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
Attributes = ComputeFileAttributes(parameters, lbv?.Dependency),
Dependency = new DependencyGraph(),
};
cb.Versions.Add(cbv);
var context = new IncrementalBuildContext(baseDir, lastBaseDir, lastBuildStartTime, canBuildInfoIncremental, parameters, cbv, lbv);
return context;
}
private void BuildDocument(
FileCollection files,
string inputFolder,
string outputFolder,
Dictionary <string, object> metadata = null,
ApplyTemplateSettings applyTemplateSettings = null,
string templateFolder = null,
string intermediateFolder = null,
Dictionary <string, ChangeKindWithDependency> changes = null)
{
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray <string> .Empty, null, intermediateFolder))
{
if (applyTemplateSettings == null)
{
applyTemplateSettings = new ApplyTemplateSettings(inputFolder, outputFolder);
}
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Directory.GetCurrentDirectory(), outputFolder),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = metadata?.ToImmutableDictionary(),
TemplateManager = new TemplateManager(null, null, new List <string> {
templateFolder
}, null, null),
TemplateDir = templateFolder,
Changes = changes?.ToImmutableDictionary(),
};
builder.Build(parameters);
}
}
public void ProcessSwaggerWithMultiUidOverwriteShouldSucceed()
{
FileCollection files = new FileCollection(_defaultFiles);
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
files.Add(DocumentType.Overwrite, new[] { "TestData/overwrite/rest.overwrite.multi.uid.md" });
files.Add(DocumentType.Overwrite, new[] { "TestData/overwrite/rest.overwrite.unmergable.md" });
parameters.Files = files;
using (var builder = new DocumentBuilder())
{
builder.Build(parameters);
}
{
var outputRawModelPath = Path.Combine(_outputFolder, Path.ChangeExtension("contacts_swagger2.json", RawModelFileExtension));
Assert.True(File.Exists(outputRawModelPath));
var model = JsonUtility.Deserialize <RestApiItemViewModel>(outputRawModelPath);
Assert.Equal("graph_windows_net_myorganization_Contacts", model.HtmlId);
Assert.Equal("<p>Overwrite content1</p>\n", model.Conceptual);
Assert.Equal("<p>Overwrite "content2"</p>\n", model.Summary);
Assert.Equal("<p>Overwrite 'content3'</p>\n", model.Metadata["not_defined_property"]);
}
}
public void ProcessSwaggerWithDefaultOverwriteShouldSucceed()
{
FileCollection files = new FileCollection(_defaultFiles);
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
files.Add(DocumentType.Overwrite, new[] { "TestData/overwrite/rest.overwrite.default.md" });
parameters.Files = files;
using (var builder = new DocumentBuilder())
{
builder.Build(parameters);
}
{
var outputRawModelPath = Path.Combine(_outputFolder, Path.ChangeExtension("contacts_swagger2.json", RawModelFileExtension));
Assert.True(File.Exists(outputRawModelPath));
var model = JsonUtility.Deserialize <RestApiItemViewModel>(outputRawModelPath);
Assert.Equal("<p>Overwrite summary</p>\n", model.Summary);
Assert.Equal("<p>Overwrite content</p>\n", model.Conceptual);
}
}
private void BuildDocument(FileCollection files, Dictionary <string, object> metadata = null, ApplyTemplateSettings applyTemplateSettings = null, string templateFolder = null, string versionDir = null)
{
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray <string> .Empty, null))
{
if (applyTemplateSettings == null)
{
applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
applyTemplateSettings.RawModelExportSettings.Export = true;
}
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Directory.GetCurrentDirectory(), _outputFolder),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = metadata?.ToImmutableDictionary(),
TemplateManager = new TemplateManager(null, null, new List <string> {
_templateFolder
}, null, null),
TemplateDir = templateFolder,
VersionDir = versionDir,
XRefMaps = ImmutableArray.Create("TestData/xrefmap.yml")
};
builder.Build(parameters);
}
}
private static Dictionary <string, FileAttributeItem> ComputeFileAttributes(DocumentBuildParameters parameters, DependencyGraph dg)
{
var filesInScope = from f in parameters.Files.EnumerateFiles()
let fileKey = ((RelativePath)f.File).GetPathFromWorkingFolder().ToString()
select new
{
PathFromWorkingFolder = fileKey,
FullPath = f.FullPath
};
var files = filesInScope;
if (dg != null)
{
var filesFromDependency = from node in dg.GetAllDependentNodes()
let fullPath = Path.Combine(EnvironmentContext.BaseDirectory, ((RelativePath)node).RemoveWorkingFolder())
select new
{
PathFromWorkingFolder = node,
FullPath = fullPath
};
files = files.Concat(filesFromDependency);
}
return((from item in files
where File.Exists(item.FullPath)
group item by item.PathFromWorkingFolder into g
select new FileAttributeItem
{
File = g.Key,
LastModifiedTime = File.GetLastWriteTimeUtc(g.First().FullPath),
MD5 = StringExtension.GetMd5String(File.ReadAllText(g.First().FullPath)),
}).ToDictionary(a => a.File));
}
private static string ComputeConfigHash(DocumentBuildParameters parameter)
{
return(StringExtension.GetMd5String(JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalCheckPropertiesResolver()
})));
}
public static IncrementalBuildContext Create(DocumentBuildParameters parameters, BuildInfo cb, BuildInfo lb, string intermediateFolder, string markdownServiceContextHash)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (cb == null)
{
throw new ArgumentNullException(nameof(cb));
}
if (intermediateFolder == null)
{
throw new ArgumentNullException(nameof(intermediateFolder));
}
var baseDir = Path.Combine(intermediateFolder, cb.DirectoryName);
var lastBaseDir = lb != null?Path.Combine(intermediateFolder, lb.DirectoryName) : null;
var lastBuildStartTime = lb?.BuildStartTime;
var buildInfoIncrementalStatus = GetBuildInfoIncrementalStatus(cb, lb, parameters.ForceRebuild);
var lbv = lb?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
var cbv = new BuildVersionInfo()
{
BaseDir = Path.GetFullPath(Environment.ExpandEnvironmentVariables(baseDir)),
VersionName = parameters.VersionName,
ConfigHash = ComputeConfigHash(parameters, markdownServiceContextHash),
FileMetadataHash = ComputeFileMetadataHash(parameters.FileMetadata),
FileMetadata = parameters.FileMetadata,
AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
OutputFile = IncrementalUtility.CreateRandomFileName(baseDir),
XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
FileMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
TocRestructionsFile = IncrementalUtility.CreateRandomFileName(baseDir),
};
if (parameters.FileMetadata != null)
{
cbv.FileMetadataFile = IncrementalUtility.CreateRandomFileName(baseDir);
}
cb.Versions.Add(cbv);
var context = new IncrementalBuildContext(baseDir, lastBaseDir, lastBuildStartTime, buildInfoIncrementalStatus, parameters, cbv, lbv)
{
IsTemplateUpdated = cb.TemplateHash != lb?.TemplateHash
};
if (context.IsTemplateUpdated)
{
Logger.LogVerbose($"Cannot build incrementally in link phase because template changed (new: {cb.TemplateHash} vs. old: {lb?.TemplateHash ?? "null"}). Will apply templates and post process all files", code: InfoCodes.IncrementalBuildReason.TemplateChanged);
}
context.InitDependency();
context.InitFileAttributes();
context.InitChanges();
return(context);
}
public void Build(DocumentBuildParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.OutputBaseDir == null)
{
throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
}
if (parameters.Files == null)
{
throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
}
if (parameters.Metadata == null)
{
parameters.Metadata = ImmutableDictionary<string, object>.Empty;
}
using (new LoggerPhaseScope(Phase))
{
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Environment.CurrentDirectory, parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages);
Logger.LogInfo("Start building document ...");
foreach (var item in
from file in parameters.Files.EnumerateFiles()
group file by (from processor in Processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupportted
orderby priority descending
select processor).FirstOrDefault())
{
if (item.Key != null)
{
BuildCore(item.Key, item, parameters.Metadata, parameters.FileMetadata, context);
}
else
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.File);
}
Logger.LogWarning(sb.ToString());
}
}
context.SerializeTo(parameters.OutputBaseDir);
Logger.LogInfo("Building document completed.");
}
}
private IncrementalBuildContext(string baseDir, string lastBaseDir, DateTime?lastBuildStartTime, bool canBuildInfoIncremental, DocumentBuildParameters parameters, BuildVersionInfo cbv, BuildVersionInfo lbv)
{
_parameters = parameters;
BaseDir = baseDir;
LastBaseDir = lastBaseDir;
LastBuildStartTime = lastBuildStartTime;
CurrentBuildVersionInfo = cbv;
LastBuildVersionInfo = lbv;
CanVersionIncremental = canBuildInfoIncremental && GetCanVersionIncremental();
}
private IncrementalBuildContext(string baseDir, string lastBaseDir, DateTime?lastBuildStartTime, IncrementalStatus buildInfoIncrementalStatus, DocumentBuildParameters parameters, BuildVersionInfo cbv, BuildVersionInfo lbv)
{
_parameters = parameters;
BaseDir = baseDir;
LastBaseDir = lastBaseDir;
LastBuildStartTime = lastBuildStartTime;
CurrentBuildVersionInfo = cbv;
LastBuildVersionInfo = lbv;
IncrementalInfo = new IncrementalInfo();
CanVersionIncremental = GetCanVersionIncremental(buildInfoIncrementalStatus);
}
public IncrementalBuildContext(DocumentBuildParameters parameters, BuildVersionInfo cbv, BuildVersionInfo lbv, bool canBuildInfoIncremental)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
_parameters = parameters;
CurrentBuildVersionInfo = cbv;
LastBuildVersionInfo = lbv;
CanVersionIncremental = canBuildInfoIncremental && GetCanVersionIncremental();
}
private static DocumentBuildParameters ConfigToParameter(BuildJsonConfig config, TemplateManager templateManager, string baseDirectory, string outputDirectory)
{
var parameters = new DocumentBuildParameters();
parameters.OutputBaseDir = outputDirectory;
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
parameters.ExternalReferencePackages =
GetFilesFromFileMapping(
GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference))
.ToImmutableArray();
parameters.Files = GetFileCollectionFromFileMapping(
baseDirectory,
GlobUtility.ExpandFileMapping(baseDirectory, config.Content),
GlobUtility.ExpandFileMapping(baseDirectory, config.Overwrite),
GlobUtility.ExpandFileMapping(baseDirectory, config.Resource));
var applyTemplateSettings = new ApplyTemplateSettings(baseDirectory, outputDirectory)
{
TransformDocument = config.DryRun != true,
};
applyTemplateSettings.RawModelExportSettings.Export = config.ExportRawModel == true;
if (!string.IsNullOrEmpty(config.RawModelOutputFolder))
{
applyTemplateSettings.RawModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.RawModelOutputFolder);
}
applyTemplateSettings.ViewModelExportSettings.Export = config.ExportViewModel == true;
if (!string.IsNullOrEmpty(config.ViewModelOutputFolder))
{
applyTemplateSettings.ViewModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.ViewModelOutputFolder);
}
parameters.ApplyTemplateSettings = applyTemplateSettings;
parameters.TemplateManager = templateManager;
if (config.MaxParallelism == null || config.MaxParallelism.Value <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
else
{
parameters.MaxParallelism = config.MaxParallelism.Value;
}
return(parameters);
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
TemplateManager = _templateManager,
};
using var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray <string> .Empty, null, "obj");
builder.Build(parameters);
}
private static string ComputeConfigHash(DocumentBuildParameters parameter, string markdownServiceContextHash)
{
var json = JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalIgnorePropertiesResolver()
});
var config = json + "|" + markdownServiceContextHash;
Logger.LogVerbose($"Config content: {config}");
return(config.GetMd5String());
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
};
using (var builder = new DocumentBuilder(LoadAssemblies()))
{
builder.Build(parameters);
}
}
public void ProcessMrefWithMergeOverwriteProcessorShouldSucceed()
{
var files = new FileCollection(Directory.GetCurrentDirectory());
files.Add(DocumentType.Article, new[] { "TestData/mref/CatLibrary.Cat-2.yml" }, "TestData/");
files.Add(DocumentType.Overwrite, new[]
{
"TestData/overwrite/mref.overwrite.default.md",
"TestData/overwrite/mref.overwrite.simple.md",
});
var outputDir = GetRandomFolder();
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = outputDir,
MarkdownEngineName = "momd",
ApplyTemplateSettings = new ApplyTemplateSettings("", outputDir),
};
var assemblies = new[]
{
typeof(ManagedReferenceDocumentProcessor).Assembly,
typeof(MergeMrefOverwriteDocumentProcessor).Assembly,
};
using (var builder = new DocumentBuilder(assemblies, ImmutableArray <string> .Empty, null))
{
builder.Build(parameters);
}
var yaml = YamlUtility.Deserialize <PageViewModel>(Path.Combine(outputDir, "mref/CatLibrary.Cat-2.yml"));
Assert.Collection(
yaml.Items.Where(item => item.Uid == "CatLibrary.Cat`2.#ctor"),
e =>
{
Assert.Equal("Overwrite *markdown* summary\n\n", e.Summary);
Assert.Equal("Overwrite *markdown* content\n\n", e.Conceptual);
});
Assert.Collection(
yaml.Items.Where(item => item.Uid == "CatLibrary.Cat`2"),
e =>
{
Assert.Equal("Overwrite <b>html</b> content\n\n", e.Summary);
Assert.Equal("original conceptual", e.Conceptual);
});
}
private static string ComputeConfigHash(DocumentBuildParameters parameter, string markdownServiceContextHash)
{
using (new LoggerPhaseScope("ComputeConfigHash", LogLevel.Diagnostic))
{
var json = JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalIgnorePropertiesResolver()
});
string configStr = json + "|" + markdownServiceContextHash;
Logger.LogVerbose($"Config content: {configStr}");
return(configStr.GetMd5String());
}
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
using var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray <string> .Empty, null);
builder.Build(parameters);
}
private void BuildDocument(FileCollection files)
{
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Test UniversalReferenceDocumentProcessor",
}.ToImmutableDictionary(),
TemplateManager = _templateManager
};
using var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray <string> .Empty, null);
builder.Build(parameters);
}
public static IncrementalBuildContext Create(DocumentBuildParameters parameters, BuildInfo cb, BuildInfo lb, string intermediateFolder, string markdownServiceContextHash)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (cb == null)
{
throw new ArgumentNullException(nameof(cb));
}
if (intermediateFolder == null)
{
throw new ArgumentNullException(nameof(intermediateFolder));
}
var baseDir = Path.Combine(intermediateFolder, cb.DirectoryName);
var lastBaseDir = lb != null?Path.Combine(intermediateFolder, lb.DirectoryName) : null;
var lastBuildStartTime = lb?.BuildStartTime;
var buildInfoIncrementalStatus = GetBuildInfoIncrementalStatus(cb, lb);
var lbv = lb?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
var cbv = new BuildVersionInfo()
{
BaseDir = Path.GetFullPath(Environment.ExpandEnvironmentVariables(baseDir)),
VersionName = parameters.VersionName,
ConfigHash = ComputeConfigHash(parameters, markdownServiceContextHash),
AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
OutputFile = IncrementalUtility.CreateRandomFileName(baseDir),
XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
FileMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
TocRestructionsFile = IncrementalUtility.CreateRandomFileName(baseDir),
};
cb.Versions.Add(cbv);
var context = new IncrementalBuildContext(baseDir, lastBaseDir, lastBuildStartTime, buildInfoIncrementalStatus, parameters, cbv, lbv)
{
IsTemplateUpdated = (cb.TemplateHash != lb?.TemplateHash)
};
context.InitDependency();
context.InitFileAttributes();
context.InitChanges();
return(context);
}
public void TestRestApiDocumentProcessorWithSwaggerJsonFile()
{
const string RawModelFileExtension = ".raw.json";
// TODO: Add override markdown file
FileCollection files = new FileCollection(Environment.CurrentDirectory);
files.Add(DocumentType.Article, new[] { "TestData/contacts_swagger2.json" }, p => (((RelativePath)p) - (RelativePath)"TestData/").ToString());
using (var builder = new DocumentBuilder())
{
var applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
applyTemplateSettings.RawModelExportSettings.Export = true;
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
builder.Build(parameters);
}
// Check REST API
var outputRawModelPath = Path.Combine(_outputFolder, Path.ChangeExtension("contacts_swagger2.json", RawModelFileExtension));
Assert.True(File.Exists(outputRawModelPath));
var model = RestApiDocumentProcessor.GetModelWithoutRef <RestApiItemViewModel>(File.ReadAllText(outputRawModelPath));
Assert.Equal("graph.windows.net/myorganization/Contacts", model.Uid);
Assert.Equal("graph_windows_net_myorganization_Contacts", model.HtmlId);
Assert.Equal(9, model.Children.Count);
Assert.Equal("Hello world!", model.Metadata["meta"]);
var item1 = model.Children[0];
Assert.Equal("graph.windows.net/myorganization/Contacts/get contacts", item1.Uid);
Assert.Equal("<p>You can get a collection of contacts from your tenant.</p>\n", item1.Summary);
Assert.Equal(1, item1.Parameters.Count);
Assert.Equal("1.6", item1.Parameters[0].Metadata["default"]);
Assert.Equal(1, item1.Responses.Count);
Assert.Equal("200", item1.Responses[0].HttpStatusCode);
}
public static IncrementalBuildContext Create(DocumentBuildParameters parameters, BuildInfo cb, BuildInfo lb, string intermediateFolder, string markdownServiceContextHash)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (cb == null)
{
throw new ArgumentNullException(nameof(cb));
}
if (intermediateFolder == null)
{
throw new ArgumentNullException(nameof(intermediateFolder));
}
var baseDir = Path.Combine(intermediateFolder, cb.DirectoryName);
var lastBaseDir = lb != null?Path.Combine(intermediateFolder, lb.DirectoryName) : null;
var lastBuildStartTime = lb?.BuildStartTime;
var canBuildInfoIncremental = CanBuildInfoIncremental(cb, lb);
var lbv = lb?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
var cbv = new BuildVersionInfo
{
VersionName = parameters.VersionName,
ConfigHash = ComputeConfigHash(parameters, markdownServiceContextHash),
AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
OutputFile = IncrementalUtility.CreateRandomFileName(baseDir),
XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
FileMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
Attributes = ComputeFileAttributes(parameters, lbv?.Dependency),
Dependency = ConstructDependencyGraphFromLast(lbv?.Dependency),
};
cb.Versions.Add(cbv);
var context = new IncrementalBuildContext(baseDir, lastBaseDir, lastBuildStartTime, canBuildInfoIncremental, parameters, cbv, lbv);
context.InitChanges();
return(context);
}
public void ProcessSwaggerWithInvalidOverwriteShouldFail()
{
FileCollection files = new FileCollection(_defaultFiles);
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
files.Add(DocumentType.Overwrite, new[] { "TestData/overwrite/rest.overwrite.invalid.md" });
parameters.Files = files;
using (var builder = new DocumentBuilder())
{
Assert.Throws <DocumentException>(() => builder.Build(parameters));
}
}
private static DocumentBuildParameters ConfigToParameter(BuildJsonConfig config)
{
var parameters = new DocumentBuildParameters();
var baseDirectory = config.BaseDirectory ?? Environment.CurrentDirectory;
parameters.OutputBaseDir = Path.Combine(baseDirectory, "obj");
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
parameters.ExternalReferencePackages = GetFilesFromFileMapping(GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference)).ToImmutableArray();
parameters.Files = GetFileCollectionFromFileMapping(baseDirectory,
Tuple.Create(DocumentType.Article, GlobUtility.ExpandFileMapping(baseDirectory, config.Content)),
Tuple.Create(DocumentType.Override, GlobUtility.ExpandFileMapping(baseDirectory, config.Overwrite)),
Tuple.Create(DocumentType.Resource, GlobUtility.ExpandFileMapping(baseDirectory, config.Resource)));
return(parameters);
}
private void BuildDocument(FileCollection files, Dictionary <string, object> metadata = null, ApplyTemplateSettings applyTemplateSettings = null)
{
using (var builder = new DocumentBuilder(LoadAssemblies()))
{
if (applyTemplateSettings == null)
{
applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
applyTemplateSettings.RawModelExportSettings.Export = true;
}
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Environment.CurrentDirectory, _outputFolder),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = metadata?.ToImmutableDictionary(),
TemplateManager = new TemplateManager(null, null, new List <string> {
_templateFolder
}, null, null)
};
builder.Build(parameters);
}
}
private static DocumentBuildParameters ConfigToParameter(BuildJsonConfig config, string baseDirectory, string outputDirectory)
{
var parameters = new DocumentBuildParameters();
parameters.OutputBaseDir = outputDirectory;
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
parameters.ExternalReferencePackages = GetFilesFromFileMapping(GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference)).ToImmutableArray();
parameters.Files = GetFileCollectionFromFileMapping(
baseDirectory,
Tuple.Create(DocumentType.Article, GlobUtility.ExpandFileMapping(baseDirectory, config.Content)),
Tuple.Create(DocumentType.Override, GlobUtility.ExpandFileMapping(baseDirectory, config.Overwrite)),
Tuple.Create(DocumentType.Resource, GlobUtility.ExpandFileMapping(baseDirectory, config.Resource)));
parameters.ExportViewModel = config.ExportViewModel == true;
parameters.ExportRawModel = config.ExportRawModel == true;
return(parameters);
}
public void ProcessSwaggerhouldSucceed()
{
FileCollection files = new FileCollection(_defaultFiles);
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = _outputFolder,
ApplyTemplateSettings = _applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
using (var builder = new DocumentBuilder())
{
builder.Build(parameters);
}
var outputRawModelPath = Path.Combine(_outputFolder, Path.ChangeExtension("contacts_swagger2.json", RawModelFileExtension));
Assert.True(File.Exists(outputRawModelPath));
var model = JsonUtility.Deserialize <RestApiItemViewModel>(outputRawModelPath);
Assert.Equal("graph.windows.net/myorganization/Contacts", model.Uid);
Assert.Equal("graph_windows_net_myorganization_Contacts", model.HtmlId);
Assert.Equal(9, model.Children.Count);
Assert.Equal("Hello world!", model.Metadata["meta"]);
var item1 = model.Children[0];
Assert.Equal("graph.windows.net/myorganization/Contacts/get contacts", item1.Uid);
Assert.Equal("<p>You can get a collection of contacts from your tenant.</p>\n", item1.Summary);
Assert.Equal(1, item1.Parameters.Count);
Assert.Equal("1.6", item1.Parameters[0].Metadata["default"]);
Assert.Equal(1, item1.Responses.Count);
Assert.Equal("200", item1.Responses[0].HttpStatusCode);
}
private static IEnumerable <DocumentBuildParameters> ConfigToParameter(BuildJsonConfig config, TemplateManager templateManager, string baseDirectory, string outputDirectory, string templateDir)
{
var parameters = new DocumentBuildParameters();
parameters.OutputBaseDir = outputDirectory;
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
if (config.PostProcessors != null)
{
parameters.PostProcessors = config.PostProcessors.ToImmutableArray();
}
parameters.ExternalReferencePackages =
GetFilesFromFileMapping(
GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference))
.ToImmutableArray();
if (config.XRefMaps != null)
{
parameters.XRefMaps = config.XRefMaps.ToImmutableArray();
}
if (!config.NoLangKeyword)
{
parameters.XRefMaps = parameters.XRefMaps.Add("embedded:docfx/langwordMapping.yml");
}
var applyTemplateSettings = new ApplyTemplateSettings(baseDirectory, outputDirectory)
{
TransformDocument = config.DryRun != true,
};
applyTemplateSettings.RawModelExportSettings.Export = config.ExportRawModel == true;
if (!string.IsNullOrEmpty(config.RawModelOutputFolder))
{
applyTemplateSettings.RawModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.RawModelOutputFolder);
}
applyTemplateSettings.ViewModelExportSettings.Export = config.ExportViewModel == true;
if (!string.IsNullOrEmpty(config.ViewModelOutputFolder))
{
applyTemplateSettings.ViewModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.ViewModelOutputFolder);
}
parameters.ApplyTemplateSettings = applyTemplateSettings;
parameters.TemplateManager = templateManager;
if (config.MaxParallelism == null || config.MaxParallelism.Value <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
else
{
parameters.MaxParallelism = config.MaxParallelism.Value;
}
if (config.MarkdownEngineName != null)
{
parameters.MarkdownEngineName = config.MarkdownEngineName;
}
if (config.MarkdownEngineProperties != null)
{
parameters.MarkdownEngineParameters = config.MarkdownEngineProperties.ToImmutableDictionary();
}
parameters.TemplateDir = templateDir;
ChangeList changeList = null;
if (config.ChangesFile != null)
{
changeList = ChangeList.Parse(config.ChangesFile, config.BaseDirectory);
}
var fileMappingParametersDictionary = GroupFileMappings(config.Content, config.Overwrite, config.Resource);
foreach (var pair in fileMappingParametersDictionary)
{
parameters.Files = GetFileCollectionFromFileMapping(
baseDirectory,
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Content)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Overwrite)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Resource)));
parameters.VersionName = pair.Key;
parameters.Changes = GetIntersectChanges(parameters.Files, changeList);
yield return(parameters);
}
}
private static DocumentBuildParameters ConfigToParameter(BuildJsonConfig config, string baseDirectory, string outputDirectory)
{
var parameters = new DocumentBuildParameters();
parameters.OutputBaseDir = outputDirectory;
if (config.GlobalMetadata != null) parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
if (config.FileMetadata != null) parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
parameters.ExternalReferencePackages = GetFilesFromFileMapping(GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference)).ToImmutableArray();
parameters.Files = GetFileCollectionFromFileMapping(
baseDirectory,
Tuple.Create(DocumentType.Article, GlobUtility.ExpandFileMapping(baseDirectory, config.Content)),
Tuple.Create(DocumentType.Override, GlobUtility.ExpandFileMapping(baseDirectory, config.Overwrite)),
Tuple.Create(DocumentType.Resource, GlobUtility.ExpandFileMapping(baseDirectory, config.Resource)));
parameters.ExportViewModel = config.ExportViewModel == true;
parameters.ExportRawModel = config.ExportRawModel == true;
return parameters;
}
private IncrementalBuildContext(string baseDir, string lastBaseDir, DateTime? lastBuildStartTime, bool canBuildInfoIncremental, DocumentBuildParameters parameters, BuildVersionInfo cbv, BuildVersionInfo lbv)
{
_parameters = parameters;
BaseDir = baseDir;
LastBaseDir = lastBaseDir;
LastBuildStartTime = lastBuildStartTime;
CurrentBuildVersionInfo = cbv;
LastBuildVersionInfo = lbv;
CanVersionIncremental = canBuildInfoIncremental && GetCanVersionIncremental();
}
private void BuildDocument(
FileCollection files,
string inputFolder,
string outputFolder,
Dictionary<string, object> metadata = null,
ApplyTemplateSettings applyTemplateSettings = null,
string templateFolder = null,
string intermediateFolder = null,
Dictionary<string, ChangeKindWithDependency> changes = null)
{
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null, intermediateFolder))
{
if (applyTemplateSettings == null)
{
applyTemplateSettings = new ApplyTemplateSettings(inputFolder, outputFolder);
}
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Directory.GetCurrentDirectory(), outputFolder),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = metadata?.ToImmutableDictionary(),
TemplateManager = new TemplateManager(null, null, new List<string> { templateFolder }, null, null),
TemplateDir = templateFolder,
Changes = changes?.ToImmutableDictionary(),
};
builder.Build(parameters);
}
}
private static Dictionary<string, FileAttributeItem> ComputeFileAttributes(DocumentBuildParameters parameters, DependencyGraph dg)
{
var filesInScope = from f in parameters.Files.EnumerateFiles()
let fileKey = ((RelativePath)f.File).GetPathFromWorkingFolder().ToString()
select new
{
PathFromWorkingFolder = fileKey,
FullPath = f.FullPath
};
var files = filesInScope;
if (dg != null)
{
var filesFromDependency = from node in dg.GetAllDependentNodes()
let fullPath = Path.Combine(EnvironmentContext.BaseDirectory, ((RelativePath)node).RemoveWorkingFolder())
select new
{
PathFromWorkingFolder = node,
FullPath = fullPath
};
files = files.Concat(filesFromDependency);
}
return (from item in files
where File.Exists(item.FullPath)
group item by item.PathFromWorkingFolder into g
select new FileAttributeItem
{
File = g.Key,
LastModifiedTime = File.GetLastWriteTimeUtc(g.First().FullPath),
MD5 = StringExtension.GetMd5String(File.ReadAllText(g.First().FullPath)),
}).ToDictionary(a => a.File);
}
private static string ComputeConfigHash(DocumentBuildParameters parameter)
{
return StringExtension.GetMd5String(JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalCheckPropertiesResolver()
}));
}
public void TestBuild()
{
const string documentsBaseDir = "db.documents";
const string outputBaseDir = "db.output";
const string RawModelFileExtension = ".raw.json";
#region Prepare test data
if (Directory.Exists(documentsBaseDir))
{
Directory.Delete(documentsBaseDir, true);
}
if (Directory.Exists(outputBaseDir))
{
Directory.Delete(outputBaseDir, true);
}
Directory.CreateDirectory(documentsBaseDir);
Directory.CreateDirectory(documentsBaseDir + "/test");
Directory.CreateDirectory(outputBaseDir);
var tocFile = Path.Combine(documentsBaseDir, "toc.md");
var conceptualFile = Path.Combine(documentsBaseDir, "test.md");
var conceptualFile2 = Path.Combine(documentsBaseDir, "test/test.md");
var resourceFile = Path.GetFileName(typeof(DocumentBuilderTest).Assembly.Location);
var resourceMetaFile = resourceFile + ".meta";
File.WriteAllLines(
tocFile,
new[]
{
"# [test1](test.md)",
"## [test2](test/test.md)",
"# Api",
"## [Console](@System.Console)",
"## [ConsoleColor](xref:System.ConsoleColor)",
});
File.WriteAllLines(
conceptualFile,
new[]
{
"---",
"uid: XRef1",
"a: b",
"b:",
" c: e",
"---",
"# Hello World",
"Test XRef: @XRef1",
"Test link: [link text](test/test.md)",
"Test link: [link text 2](../" + resourceFile + ")",
"<p>",
"test",
});
File.WriteAllLines(
conceptualFile2,
new[]
{
"---",
"uid: XRef2",
"a: b",
"b:",
" c: e",
"---",
"# Hello World",
"Test XRef: @XRef2",
"Test link: [link text](../test.md)",
"<p>",
"test",
});
File.WriteAllText(resourceMetaFile, @"{ abc: ""xyz"", uid: ""r1"" }");
File.WriteAllText(MarkdownSytleConfig.MarkdownStyleFileName, @"{
rules : [
""foo"",
{ name: ""bar"", disable: true}
],
tagRules : [
{
tagNames: [""p""],
behavior: ""Warning"",
messageFormatter: ""Tag {0} is not valid."",
openingTagOnly: true
}
]
}");
FileCollection files = new FileCollection(Environment.CurrentDirectory);
files.Add(DocumentType.Article, new[] { tocFile, conceptualFile, conceptualFile2 });
files.Add(DocumentType.Article, new[] { "TestData/System.Console.csyml", "TestData/System.ConsoleColor.csyml" }, p => (((RelativePath)p) - (RelativePath)"TestData/").ToString());
files.Add(DocumentType.Resource, new[] { resourceFile });
#endregion
Init(string.Join(".", nameof(DocumentBuilderTest), DocumentBuilder.PhaseName, MarkdownValidatorBuilder.MarkdownValidatePhaseName));
try
{
using (new LoggerPhaseScope(nameof(DocumentBuilderTest)))
using (var builder = new DocumentBuilder())
{
var applyTemplateSettings = new ApplyTemplateSettings(documentsBaseDir, outputBaseDir);
applyTemplateSettings.RawModelExportSettings.Export = true;
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Environment.CurrentDirectory, outputBaseDir),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = new Dictionary <string, object>
{
["meta"] = "Hello world!",
}.ToImmutableDictionary()
};
builder.Build(parameters);
}
{
// check log for markdown stylecop.
Assert.Equal(2, Listener.Items.Count);
Assert.Equal("Tag p is not valid.", Listener.Items[0].Message);
Assert.Equal(LogLevel.Warning, Listener.Items[0].LogLevel);
Assert.Equal("Tag p is not valid.", Listener.Items[1].Message);
Assert.Equal(LogLevel.Warning, Listener.Items[1].LogLevel);
}
{
// check toc.
Assert.True(File.Exists(Path.Combine(outputBaseDir, Path.ChangeExtension(tocFile, RawModelFileExtension))));
var model = JsonUtility.Deserialize <TocViewModel>(Path.Combine(outputBaseDir, Path.ChangeExtension(tocFile, RawModelFileExtension)));
Assert.NotNull(model);
Assert.Equal("test1", model[0].Name);
Assert.Equal("test.json", model[0].Href);
Assert.NotNull(model[0].Items);
Assert.Equal("test2", model[0].Items[0].Name);
Assert.Equal("test/test.json", model[0].Items[0].Href);
Assert.Equal("Api", model[1].Name);
Assert.Null(model[1].Href);
Assert.NotNull(model[1].Items);
Assert.Equal("Console", model[1].Items[0].Name);
Assert.Equal("../System.Console.json", model[1].Items[0].Href);
Assert.Equal("ConsoleColor", model[1].Items[1].Name);
Assert.Equal("../System.ConsoleColor.json", model[1].Items[1].Href);
}
{
// check conceptual.
Assert.True(File.Exists(Path.Combine(outputBaseDir, Path.ChangeExtension(conceptualFile, RawModelFileExtension))));
var model = JsonUtility.Deserialize <Dictionary <string, object> >(Path.Combine(outputBaseDir, Path.ChangeExtension(conceptualFile, RawModelFileExtension)));
Assert.Equal(
"<h1 id=\"hello-world\">Hello World</h1>",
model["rawTitle"]);
Assert.Equal(
"\n<p>Test XRef: <xref href=\"XRef1\" data-throw-if-not-resolved=\"False\" data-raw=\"@XRef1\"></xref>\n" +
"Test link: <a href=\"~/db.documents/test/test.md\">link text</a>\n" +
"Test link: <a href=\"~/" + resourceFile + "\">link text 2</a></p>\n" +
"<p><p>\n" +
"test</p>\n",
model[Constants.PropertyName.Conceptual]);
Assert.Equal("Conceptual", model["type"]);
Assert.Equal("Hello world!", model["meta"]);
Assert.Equal("b", model["a"]);
}
{
// check mref.
Assert.True(File.Exists(Path.Combine(outputBaseDir, Path.ChangeExtension("System.Console.csyml", RawModelFileExtension))));
Assert.True(File.Exists(Path.Combine(outputBaseDir, Path.ChangeExtension("System.ConsoleColor.csyml", RawModelFileExtension))));
}
{
// check resource.
Assert.True(File.Exists(Path.Combine(outputBaseDir, resourceFile)));
Assert.True(File.Exists(Path.Combine(outputBaseDir, resourceFile + RawModelFileExtension)));
var meta = JsonUtility.Deserialize <Dictionary <string, object> >(Path.Combine(outputBaseDir, resourceFile + RawModelFileExtension));
Assert.Equal(3, meta.Count);
Assert.True(meta.ContainsKey("meta"));
Assert.Equal("Hello world!", meta["meta"]);
Assert.True(meta.ContainsKey("abc"));
Assert.Equal("xyz", meta["abc"]);
Assert.True(meta.ContainsKey(Constants.PropertyName.Uid));
Assert.Equal("r1", meta[Constants.PropertyName.Uid]);
}
}
finally
{
CleanUp();
Directory.Delete(documentsBaseDir, true);
Directory.Delete(outputBaseDir, true);
File.Delete(resourceMetaFile);
}
}
private static IEnumerable <DocumentBuildParameters> ConfigToParameter(BuildJsonConfig config, TemplateManager templateManager, ChangeList changeList, string baseDirectory, string outputDirectory, string templateDir)
{
var parameters = new DocumentBuildParameters
{
OutputBaseDir = outputDirectory,
ForceRebuild = config.Force ?? false,
ForcePostProcess = config.ForcePostProcess ?? false
};
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
if (config.PostProcessors != null)
{
parameters.PostProcessors = config.PostProcessors.ToImmutableArray();
}
parameters.ExternalReferencePackages =
GetFilesFromFileMapping(
GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference))
.ToImmutableArray();
if (config.XRefMaps != null)
{
parameters.XRefMaps = config.XRefMaps.ToImmutableArray();
}
if (!config.NoLangKeyword)
{
parameters.XRefMaps = parameters.XRefMaps.Add("embedded:docfx/langwordMapping.yml");
}
string outputFolderForDebugFiles = null;
if (!string.IsNullOrEmpty(config.OutputFolderForDebugFiles))
{
outputFolderForDebugFiles = Path.Combine(baseDirectory, config.OutputFolderForDebugFiles);
}
var applyTemplateSettings = new ApplyTemplateSettings(baseDirectory, outputDirectory, outputFolderForDebugFiles, config.EnableDebugMode ?? false)
{
TransformDocument = config.DryRun != true,
};
applyTemplateSettings.RawModelExportSettings.Export = config.ExportRawModel == true;
if (!string.IsNullOrEmpty(config.RawModelOutputFolder))
{
applyTemplateSettings.RawModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.RawModelOutputFolder);
}
applyTemplateSettings.ViewModelExportSettings.Export = config.ExportViewModel == true;
if (!string.IsNullOrEmpty(config.ViewModelOutputFolder))
{
applyTemplateSettings.ViewModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.ViewModelOutputFolder);
}
parameters.ApplyTemplateSettings = applyTemplateSettings;
parameters.TemplateManager = templateManager;
if (config.MaxParallelism == null || config.MaxParallelism.Value <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
else
{
parameters.MaxParallelism = config.MaxParallelism.Value;
int wt, cpt;
ThreadPool.GetMinThreads(out wt, out cpt);
if (wt < parameters.MaxParallelism)
{
ThreadPool.SetMinThreads(parameters.MaxParallelism, cpt);
}
}
if (config.MarkdownEngineName != null)
{
parameters.MarkdownEngineName = config.MarkdownEngineName;
}
if (config.MarkdownEngineProperties != null)
{
parameters.MarkdownEngineParameters = config.MarkdownEngineProperties.ToImmutableDictionary();
}
if (config.CustomLinkResolver != null)
{
parameters.CustomLinkResolver = config.CustomLinkResolver;
}
parameters.TemplateDir = templateDir;
var fileMappingParametersDictionary = GroupFileMappings(config.Content, config.Overwrite, config.Resource);
if (config.LruSize == null)
{
parameters.LruSize = Environment.Is64BitProcess ? 0x2000 : 0xC00;
}
else
{
parameters.LruSize = Math.Max(0, config.LruSize.Value);
}
foreach (var pair in fileMappingParametersDictionary)
{
var p = parameters.Clone();
VersionConfig vi;
if (config.Versions != null && config.Versions.TryGetValue(pair.Key, out vi))
{
if (!string.IsNullOrEmpty(vi.Destination))
{
p.VersionDir = vi.Destination;
}
}
p.Files = GetFileCollectionFromFileMapping(
baseDirectory,
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Content)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Overwrite)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Resource)));
p.VersionName = pair.Key;
p.Changes = GetIntersectChanges(p.Files, changeList);
// TODO: move RootTocPath to VersionInfo
p.RootTocPath = pair.Value.RootTocPath;
yield return(p);
}
}
public void Build(DocumentBuildParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.OutputBaseDir == null)
{
throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
}
if (parameters.Files == null)
{
throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
}
if (parameters.Metadata == null)
{
parameters.Metadata = ImmutableDictionary<string, object>.Empty;
}
using (new LoggerPhaseScope(PhaseName))
{
Directory.CreateDirectory(parameters.OutputBaseDir);
var context = new DocumentBuildContext(
Path.Combine(Environment.CurrentDirectory, parameters.OutputBaseDir),
parameters.Files.EnumerateFiles(),
parameters.ExternalReferencePackages,
parameters.TemplateCollection
);
Logger.LogInfo("Start building document...");
IEnumerable<InnerBuildContext> innerContexts = Enumerable.Empty<InnerBuildContext>();
try
{
innerContexts = GetInnerContexts(parameters, Processors).ToList();
foreach (var item in innerContexts)
{
BuildCore(item.HostService, item.Processor, context);
}
foreach (var item in innerContexts)
{
UpdateHref(item.HostService, item.Processor, context);
}
UpdateContext(context);
if (parameters.ExportRawModel)
{
Logger.LogInfo("Start exporting raw model...");
foreach(var item in context.Manifest)
{
var model = item.Model;
if (model.Content != null)
{
var rawModelPath = Path.Combine(model.BaseDir, Path.ChangeExtension(model.File, RawModelExtension));
JsonUtility.Serialize(rawModelPath, model.Content);
}
}
}
Transform(context, parameters.TemplateCollection, parameters.ExportViewModel);
}
finally
{
foreach (var item in innerContexts)
{
if (item.HostService != null)
{
Cleanup(item.HostService);
item.HostService.Dispose();
}
}
}
Logger.LogInfo("Building document completed.");
}
}
private static string ComputeConfigHash(DocumentBuildParameters parameter, string markdownServiceContextHash)
{
var json = JsonConvert.SerializeObject(
parameter,
new JsonSerializerSettings
{
ContractResolver = new IncrementalIgnorePropertiesResolver()
});
var config = json + "|" + markdownServiceContextHash;
Logger.LogVerbose($"Config content: {config}");
return config.GetMd5String();
}
private static IEnumerable <DocumentBuildParameters> ConfigToParameter(BuildJsonConfig config, TemplateManager templateManager, ChangeList changeList, string baseDirectory, string outputDirectory, string templateDir)
{
var parameters = new DocumentBuildParameters
{
OutputBaseDir = outputDirectory,
ForceRebuild = config.Force ?? false,
ForcePostProcess = config.ForcePostProcess ?? false,
SitemapOptions = config.SitemapOptions,
FALName = config.FALName,
DisableGitFeatures = config.DisableGitFeatures,
SchemaLicense = config.SchemaLicense,
TagParameters = config.TagParameters
};
if (config.GlobalMetadata != null)
{
parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
}
if (config.FileMetadata != null)
{
parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
}
if (config.PostProcessors != null)
{
parameters.PostProcessors = config.PostProcessors.ToImmutableArray();
}
if (config.XRefMaps != null)
{
parameters.XRefMaps = config.XRefMaps.ToImmutableArray();
}
if (config.XRefServiceUrls != null)
{
parameters.XRefServiceUrls = config.XRefServiceUrls.ToImmutableArray();
}
if (!config.NoLangKeyword)
{
parameters.XRefMaps = parameters.XRefMaps.Add("embedded:docfx/langwordMapping.yml");
}
string outputFolderForDebugFiles = null;
if (!string.IsNullOrEmpty(config.OutputFolderForDebugFiles))
{
outputFolderForDebugFiles = Path.Combine(baseDirectory, config.OutputFolderForDebugFiles);
}
var applyTemplateSettings = new ApplyTemplateSettings(baseDirectory, outputDirectory, outputFolderForDebugFiles, config.EnableDebugMode ?? false)
{
TransformDocument = config.DryRun != true,
};
applyTemplateSettings.RawModelExportSettings.Export = config.ExportRawModel == true;
if (!string.IsNullOrEmpty(config.RawModelOutputFolder))
{
applyTemplateSettings.RawModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.RawModelOutputFolder);
}
applyTemplateSettings.ViewModelExportSettings.Export = config.ExportViewModel == true;
if (!string.IsNullOrEmpty(config.ViewModelOutputFolder))
{
applyTemplateSettings.ViewModelExportSettings.OutputFolder = Path.Combine(baseDirectory, config.ViewModelOutputFolder);
}
parameters.ApplyTemplateSettings = applyTemplateSettings;
parameters.TemplateManager = templateManager;
if (config.MaxParallelism == null || config.MaxParallelism.Value <= 0)
{
parameters.MaxParallelism = Environment.ProcessorCount;
}
else
{
parameters.MaxParallelism = config.MaxParallelism.Value;
ThreadPool.GetMinThreads(out int wt, out int cpt);
if (wt < parameters.MaxParallelism)
{
ThreadPool.SetMinThreads(parameters.MaxParallelism, cpt);
}
}
parameters.MaxHttpParallelism = Math.Max(64, parameters.MaxParallelism * 2);
ServicePointManager.DefaultConnectionLimit = parameters.MaxHttpParallelism;
if (config.MarkdownEngineName != null)
{
parameters.MarkdownEngineName = config.MarkdownEngineName;
}
if (config.MarkdownEngineProperties != null)
{
parameters.MarkdownEngineParameters = config.MarkdownEngineProperties.ToImmutableDictionary();
}
if (config.CustomLinkResolver != null)
{
parameters.CustomLinkResolver = config.CustomLinkResolver;
}
parameters.TemplateDir = templateDir;
var fileMappingParametersDictionary = GroupFileMappings(config.Content, config.Overwrite, config.Resource);
if (config.LruSize == null)
{
parameters.LruSize = Environment.Is64BitProcess ? 0x2000 : 0xC00;
}
else
{
parameters.LruSize = Math.Max(0, config.LruSize.Value);
}
if (config.KeepFileLink)
{
parameters.KeepFileLink = true;
}
if (config.Pairing != null)
{
parameters.OverwriteFragmentsRedirectionRules = config.Pairing.Select(i => new FolderRedirectionRule(i.ContentFolder, i.OverwriteFragmentsFolder)).ToImmutableArray();
}
foreach (var pair in fileMappingParametersDictionary)
{
var p = parameters.Clone();
if (!string.IsNullOrEmpty(pair.Key))
{
p.GroupInfo = new GroupInfo()
{
Name = pair.Key,
};
if (config.Groups != null && config.Groups.TryGetValue(pair.Key, out GroupConfig gi))
{
p.GroupInfo.Destination = gi.Destination;
p.GroupInfo.Metadata = gi.Metadata;
if (!string.IsNullOrEmpty(gi.Destination))
{
p.VersionDir = gi.Destination;
}
}
}
p.Files = GetFileCollectionFromFileMapping(
baseDirectory,
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Content)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Overwrite)),
GlobUtility.ExpandFileMapping(baseDirectory, pair.Value.GetFileMapping(FileMappingType.Resource)));
p.VersionName = pair.Key;
p.Changes = GetIntersectChanges(p.Files, changeList);
p.RootTocPath = pair.Value.RootTocPath;
yield return(p);
}
}
private static IEnumerable<InnerBuildContext> GetInnerContexts(DocumentBuildParameters parameters, IEnumerable<IDocumentProcessor> processors)
{
var filesGroupedByProcessor =
from file in parameters.Files.EnumerateFiles()
group file by (from processor in processors
let priority = processor.GetProcessingPriority(file)
where priority != ProcessingPriority.NotSupportted
orderby priority descending
select processor).FirstOrDefault();
var toHandleItems = filesGroupedByProcessor.Where(s => s.Key != null);
var notToHandleItems = filesGroupedByProcessor.Where(s => s.Key == null);
foreach (var item in notToHandleItems)
{
var sb = new StringBuilder();
sb.AppendLine("Cannot handle following file:");
foreach (var f in item)
{
sb.Append("\t");
sb.AppendLine(f.File);
}
Logger.LogWarning(sb.ToString());
}
foreach (var item in toHandleItems)
{
yield return new InnerBuildContext(new HostService(
from file in item
select Load(item.Key, parameters.Metadata, parameters.FileMetadata, file)),
item.Key);
}
}
private void BuildDocument(FileCollection files, Dictionary<string, object> metadata = null, ApplyTemplateSettings applyTemplateSettings = null, string templateFolder = null)
{
using (var builder = new DocumentBuilder(LoadAssemblies(), ImmutableArray<string>.Empty, null, templateFolder))
{
if (applyTemplateSettings == null)
{
applyTemplateSettings = new ApplyTemplateSettings(_inputFolder, _outputFolder);
applyTemplateSettings.RawModelExportSettings.Export = true;
}
var parameters = new DocumentBuildParameters
{
Files = files,
OutputBaseDir = Path.Combine(Directory.GetCurrentDirectory(), _outputFolder),
ApplyTemplateSettings = applyTemplateSettings,
Metadata = metadata?.ToImmutableDictionary(),
TemplateManager = new TemplateManager(null, null, new List<string> { _templateFolder }, null, null),
TemplateDir = templateFolder,
};
builder.Build(parameters);
}
}
private static DocumentBuildParameters ConfigToParameter(BuildJsonConfig config)
{
var parameters = new DocumentBuildParameters();
var baseDirectory = config.BaseDirectory ?? Environment.CurrentDirectory;
parameters.OutputBaseDir = Path.Combine(baseDirectory, "obj");
if (config.GlobalMetadata != null) parameters.Metadata = config.GlobalMetadata.ToImmutableDictionary();
if (config.FileMetadata != null) parameters.FileMetadata = ConvertToFileMetadataItem(baseDirectory, config.FileMetadata);
parameters.ExternalReferencePackages = GetFilesFromFileMapping(GlobUtility.ExpandFileMapping(baseDirectory, config.ExternalReference)).ToImmutableArray();
parameters.Files = GetFileCollectionFromFileMapping(baseDirectory,
Tuple.Create(DocumentType.Article, GlobUtility.ExpandFileMapping(baseDirectory, config.Content)),
Tuple.Create(DocumentType.Override, GlobUtility.ExpandFileMapping(baseDirectory, config.Overwrite)),
Tuple.Create(DocumentType.Resource, GlobUtility.ExpandFileMapping(baseDirectory, config.Resource)));
return parameters;
}
