public DocumentBuilder(
IEnumerable <Assembly> assemblies,
ImmutableArray <string> postProcessorNames,
string templateHash,
string intermediateFolder = null,
string commitFromSHA = null,
string commitToSHA = null)
{
Logger.LogVerbose("Loading plug-in...");
using (new LoggerPhaseScope("ImportPlugins", true))
{
var assemblyList = assemblies?.ToList();
_container = GetContainer(assemblyList);
_container.SatisfyImports(this);
_currentBuildInfo.CommitFromSHA = commitFromSHA;
_currentBuildInfo.CommitToSHA = commitToSHA;
if (intermediateFolder != null)
{
_currentBuildInfo.PluginHash = ComputePluginHash(assemblyList);
_currentBuildInfo.TemplateHash = templateHash;
_currentBuildInfo.DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder);
}
}
Logger.LogInfo($"{Processors.Count()} plug-in(s) loaded.");
foreach (var processor in Processors)
{
Logger.LogVerbose($"\t{processor.Name} with build steps ({string.Join(", ", from bs in processor.BuildSteps orderby bs.BuildOrder select bs.Name)})");
}
_postProcessors = GetPostProcessor(postProcessorNames);
_intermediateFolder = intermediateFolder;
_lastBuildInfo = LoadLastBuildInfo();
}
private void CopyToCurrentCache(List <ManifestItem> increItems)
{
using (new LoggerPhaseScope("CopyToCurrentCache", LogLevel.Verbose))
{
var itemsToBeCopied = from mi in increItems
from oi in mi.OutputFiles.Values
where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir)
select oi;
Parallel.ForEach(
itemsToBeCopied,
new ParallelOptions {
MaxDegreeOfParallelism = _increContext.MaxParallelism
},
item =>
{
string cachedFileName;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to current cache.
var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName);
var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName);
File.Copy(lastCachedFile, currentCachedFile);
item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName);
});
});
}
}
public void ReportDependencyFrom(FileModel currentFileModel, string from, string type)
{
if (currentFileModel == null)
{
throw new ArgumentNullException(nameof(currentFileModel));
}
if (string.IsNullOrEmpty(from))
{
throw new ArgumentNullException(nameof(from));
}
if (type == null)
{
throw new ArgumentNullException(nameof(type));
}
if (DependencyGraph == null)
{
return;
}
lock (DependencyGraph)
{
string fromKey = IncrementalUtility.GetDependencyKey(currentFileModel.OriginalFileAndType.ChangeFile((TypeForwardedToRelativePath)currentFileModel.OriginalFileAndType.File + (TypeForwardedToRelativePath)from));
string toKey = IncrementalUtility.GetDependencyKey(currentFileModel.OriginalFileAndType);
ReportDependencyCore(fromKey, toKey, toKey, type);
}
}
private void TraceIncremental(List <ManifestItem> increItems)
{
foreach (var outputRelPath in GetOutputRelativePaths(increItems))
{
string lastCachedRelPath;
if (_increContext.LastInfo == null)
{
throw new BuildCacheException("Last incremental post processor info should not be null.");
}
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}.");
}
IncrementalUtility.RetryIO(() =>
{
var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath);
var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
// Copy last cached file to current cached file
EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName));
_increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName);
});
}
}
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
// Copy when current base dir is not last base dir
if (!FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals(
IncrementalContext.BaseDir,
IncrementalContext.LastBaseDir))
{
foreach (var ofi in result.OutputFiles.Values)
{
if (ofi.LinkToPath != null &&
ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length &&
ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir, StringComparison.Ordinal) &&
(ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/'))
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
}
}
}
return(result);
}
private void PreHandle(List <HostService> hostServices)
{
foreach (var hostService in hostServices)
{
hostService.DependencyGraph = CurrentBuildVersionInfo.Dependency;
using (new LoggerPhaseScope("RegisterDependencyTypeFromProcessor", LogLevel.Verbose))
{
hostService.RegisterDependencyType();
}
if (hostService.ShouldTraceIncrementalInfo)
{
hostService.IncrementalInfos = IncrementalContext.GetModelIncrementalInfo(hostService, Phase);
}
}
var nonIncreSet = new HashSet <string>(from h in hostServices
where !h.CanIncrementalBuild
from f in h.Models
select IncrementalUtility.GetDependencyKey(f.OriginalFileAndType),
FilePathComparer.OSPlatformSensitiveStringComparer);
ReportDependency(nonIncreSet);
LoadContextInfo(hostServices);
RegisterUnloadedTocRestructions(nonIncreSet);
Logger.RegisterListener(CurrentBuildMessageInfo.GetListener());
}
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
Parallel.ForEach(
from ofi in result.OutputFiles.Values
where ofi.LinkToPath != null
where ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length
where ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir)
where (ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/')
select ofi,
new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
},
ofi =>
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
});
return(result);
}
public void ReportDependencyFrom(FileModel currentFileModel, string from, string fromType, string type)
{
if (currentFileModel == null)
{
throw new ArgumentNullException(nameof(currentFileModel));
}
if (string.IsNullOrEmpty(from))
{
throw new ArgumentNullException(nameof(from));
}
if (fromType == null)
{
throw new ArgumentNullException(nameof(fromType));
}
if (type == null)
{
throw new ArgumentNullException(nameof(type));
}
if (DependencyGraph == null)
{
return;
}
string fromKey = fromType == DependencyItemSourceType.File ?
IncrementalUtility.GetDependencyKey(currentFileModel.OriginalFileAndType.ChangeFile((RelativePath)currentFileModel.OriginalFileAndType.File + (RelativePath)from)) :
from;
string toKey = IncrementalUtility.GetDependencyKey(currentFileModel.OriginalFileAndType);
ReportDependencyCore(new DependencyItemSourceInfo(fromType, fromKey), toKey, toKey, type);
}
private void SaveExternalXRefSpec()
{
CurrentBuildVersionInfo.ExternalXRefSpecFile = IncrementalUtility.CreateRandomFileName(CurrentBuildVersionInfo.BaseDir);
using (var writer = File.CreateText(Path.Combine(CurrentBuildVersionInfo.BaseDir, CurrentBuildVersionInfo.ExternalXRefSpecFile)))
{
Context.SaveExternalXRefSpec(writer);
}
}
private void SaveOutputs(IEnumerable <HostService> hostServices)
{
var outputDir = Context.BuildOutputFolder;
var lo = LastBuildVersionInfo?.BuildOutputs;
var outputItems = (from m in Context.ManifestItems
from output in m.OutputFiles.Values
select new
{
Path = output.RelativePath,
SourcePath = m.SourceRelativePath,
} into items
group items by items.SourcePath).ToDictionary(g => g.Key, g => g.Select(p => p.Path).ToList(), FilePathComparer.OSPlatformSensitiveStringComparer);
foreach (var h in hostServices.Where(h => h.ShouldTraceIncrementalInfo))
{
foreach (var pair in IncrementalContext.GetModelLoadInfo(h))
{
List <string> items;
if (!outputItems.TryGetValue(pair.Key, out items))
{
continue;
}
foreach (var path in items)
{
// path might be duplicate. for example, files with same name in different input folders are mapped to same output folder.
if (CurrentBuildVersionInfo.BuildOutputs.ContainsKey(path))
{
continue;
}
string fileName = IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir);
string fullPath = Path.Combine(outputDir, path);
IncrementalUtility.RetryIO(() =>
{
if (pair.Value == null)
{
if (lo == null)
{
throw new BuildCacheException($"Full build hasn't loaded build outputs.");
}
string lfn;
if (!lo.TryGetValue(path, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded output: {path}.");
}
Directory.CreateDirectory(Path.GetDirectoryName(fullPath));
File.Copy(Path.Combine(IncrementalContext.LastBaseDir, lfn), fullPath, true);
}
File.Copy(fullPath, Path.Combine(IncrementalContext.BaseDir, fileName));
CurrentBuildVersionInfo.BuildOutputs[path] = fileName;
});
}
}
}
}
public void TestIncrementalWithNullLastPostProcessInfo()
{
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
// Pass null as last build info
var postProcessors = GetPostProcessors(typeof(AppendStringPostProcessor));
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, null, postProcessors, true);
// Check context
Assert.True(increContext.ShouldTraceIncrementalInfo);
Assert.False(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.True(manifest.Files.All(f => f.IsIncremental == false));
// Check output content
VerifyOutput(outputFolder, AppendStringPostProcessor.AppendString, "a", "b", "c");
// Check cached PostProcessInfo
Assert.NotNull(currentBuildInfo.PostProcessInfo);
var postProcessorInfos = currentBuildInfo.PostProcessInfo.PostProcessorInfos;
Assert.Equal(1, currentBuildInfo.PostProcessInfo.PostProcessorInfos.Count);
Assert.Equal($"{typeof(AppendStringPostProcessor).Name}", postProcessorInfos[0].Name);
Assert.Null(postProcessorInfos[0].IncrementalContextHash);
var postProcessOutputs = currentBuildInfo.PostProcessInfo.PostProcessOutputs;
Assert.Equal(9, postProcessOutputs.Count);
VerifyCachedOutput(Path.Combine(intermediateFolder, currentBuildInfo.DirectoryName), postProcessOutputs, AppendStringPostProcessor.AppendString, AppendStringPostProcessor.AdditionalExtensionString, "a", "b", "c");
Assert.Equal <ManifestItem>(manifest.Files, currentBuildInfo.PostProcessInfo.ManifestItems);
// Check incremental info
Assert.Equal(1, manifest.IncrementalInfo.Count);
Assert.Equal(false, manifest.IncrementalInfo[0].Status.CanIncremental);
Assert.Equal(IncrementalPhase.PostProcessing, manifest.IncrementalInfo[0].Status.IncrementalPhase);
Assert.Equal("Cannot support incremental post processing, the reason is: last post processor info is null.", manifest.IncrementalInfo[0].Status.Details);
}
public Stream SaveContextInfo()
{
if (!_increContext.ShouldTraceIncrementalInfo)
{
Logger.LogVerbose("Could not save current context info since should not trace incremental information.");
return(null);
}
var currentPostProcessorInfo = FindPostProcessorInfo(_increContext.CurrentInfo, _postProcessorName);
currentPostProcessorInfo.ContextInfoFile = IncrementalUtility.CreateRandomFileName(_increContext.CurrentBaseDir);
return(EnvironmentContext.FileAbstractLayer.Create(Path.Combine(_increContext.CurrentBaseDir, currentPostProcessorInfo.ContextInfoFile)));
}
private void TraceNoneIncremental(string outputFolder, List <ManifestItem> nonIncreItems)
{
foreach (var outputRelPath in GetOutputRelativePaths(nonIncreItems, ExcludeType))
{
IncrementalUtility.RetryIO(() =>
{
var outputPath = Path.Combine(outputFolder, outputRelPath);
var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
// Copy output to current cached file
EnvironmentContext.FileAbstractLayer.Copy(outputPath, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName));
_increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName);
});
}
}
public void TestIncrementalWithNotSupportIncrementalPostProcessor()
{
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
var lastBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder),
PostProcessInfo = new PostProcessInfo()
};
lastBuildInfo.PostProcessInfo.PostProcessorInfos.Add(new PostProcessorInfo
{
Name = typeof(NonIncrementalPostProcessor).Name
});
// Add not post processor which not support incremental
var postProcessors = GetPostProcessors(typeof(NonIncrementalPostProcessor));
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.False(increContext.ShouldTraceIncrementalInfo);
Assert.False(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.True(manifest.Files.All(f => f.IsIncremental == false));
// Check output content should append nothing
VerifyOutput(outputFolder, string.Empty, "a", "b", "c");
// Check cached PostProcessInfo is null
Assert.Null(currentBuildInfo.PostProcessInfo);
// Check incremental info
Assert.Equal(1, manifest.IncrementalInfo.Count);
Assert.Equal(false, manifest.IncrementalInfo[0].Status.CanIncremental);
Assert.Equal(IncrementalPhase.PostProcessing, manifest.IncrementalInfo[0].Status.IncrementalPhase);
Assert.Equal("Cannot support incremental post processing, the reason is: should not trace intermediate info.", manifest.IncrementalInfo[0].Status.Details);
}
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext)
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
var mi = incrementalContext.GetModelLoadInfo(this);
var lmm = incrementalContext.GetLastIntermediateModelManifest(this);
var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this);
foreach (var pair in mi)
{
IncrementalUtility.RetryIO(() =>
{
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
if (pair.Value == null)
{
if (lmm == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}");
}
string lfn;
if (!lmm.Models.TryGetValue(pair.Key, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}");
}
// use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted.
File.Copy(Path.Combine(incrementalContext.LastBaseDir, lfn), Path.Combine(incrementalContext.BaseDir, fileName));
}
else
{
var key = RelativePath.NormalizedWorkingFolder + pair.Key;
var model = Models.Find(m => m.Key == key);
using (var stream = File.Create(Path.Combine(incrementalContext.BaseDir, fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
}
cmm.Models.Add(pair.Key, fileName);
});
}
}
private void CopyToOutput(List <ManifestItem> increItems, string outputFolder)
{
foreach (var outputRelPath in GetOutputRelativePaths(increItems))
{
string lastCachedRelPath;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to output
var outputPath = Path.Combine(outputFolder, outputRelPath);
var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath);
EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, outputPath);
});
}
}
public DocumentBuilder(
IEnumerable <Assembly> assemblies,
ImmutableArray <string> postProcessorNames,
string templateHash,
string intermediateFolder = null,
string commitFromSHA = null,
string commitToSHA = null)
{
Logger.LogVerbose("Loading plug-in...");
using (new LoggerPhaseScope("ImportPlugins", LogLevel.Verbose))
{
var assemblyList = assemblies?.ToList() ?? new List <Assembly>();
assemblyList.Add(typeof(DocumentBuilder).Assembly);
_container = CompositionUtility.GetContainer(assemblyList);
_container.SatisfyImports(this);
_currentBuildInfo.CommitFromSHA = commitFromSHA;
_currentBuildInfo.CommitToSHA = commitToSHA;
if (intermediateFolder != null)
{
_currentBuildInfo.PluginHash = ComputePluginHash(assemblyList);
_currentBuildInfo.TemplateHash = templateHash;
_currentBuildInfo.DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder);
}
}
Logger.LogInfo($"{Processors.Count()} plug-in(s) loaded.");
foreach (var processor in Processors)
{
Logger.LogVerbose($"\t{processor.Name} with build steps ({string.Join(", ", from bs in processor.BuildSteps orderby bs.BuildOrder select bs.Name)})");
}
if (intermediateFolder != null)
{
var expanded = Environment.ExpandEnvironmentVariables(intermediateFolder);
if (expanded.Length == 0)
{
expanded = ".";
}
_intermediateFolder = Path.GetFullPath(expanded);
}
_lastBuildInfo = BuildInfo.Load(_intermediateFolder);
_postProcessorsManager = new PostProcessorsManager(_container, postProcessorNames);
}
private DependencyGraph Load(string intermediateFolder, string versionName)
{
var expandedBaseFolder = Path.GetFullPath(Environment.ExpandEnvironmentVariables(intermediateFolder));
var buildInfoFile = Path.Combine(expandedBaseFolder, BuildInfo.FileName);
var buildInfo = JsonUtility.Deserialize <BuildInfo>(buildInfoFile);
if (buildInfo == null)
{
LogErrorAndThrow($"Cache files in the folder '{intermediateFolder}' are corrupted!", null);
}
var versionInfo = buildInfo.Versions.FirstOrDefault(v => v.VersionName == versionName);
if (versionInfo == null)
{
Logger.LogInfo($"Cache files for version '{versionName}' is not found!", null);
return(null);
}
var dependencyFile = Path.Combine(expandedBaseFolder, buildInfo.DirectoryName, versionInfo.DependencyFile);
return(IncrementalUtility.LoadDependency(dependencyFile));
}
private void ReloadDependency(IEnumerable <HostService> hostServices)
{
// restore dependency graph from last dependency graph for unchanged files
using (new LoggerPhaseScope("ReportDependencyFromLastBuild", LogLevel.Diagnostic))
{
var fileSet = new HashSet <string>(from h in hostServices
where !h.CanIncrementalBuild
from f in h.Models
select IncrementalUtility.GetDependencyKey(f.OriginalFileAndType),
FilePathComparer.OSPlatformSensitiveStringComparer);
var ldg = LastBuildVersionInfo?.Dependency;
if (ldg != null)
{
CurrentBuildVersionInfo.Dependency.ReportDependency(from r in ldg.ReportedBys
where !IncrementalContext.ChangeDict.ContainsKey(r) || IncrementalContext.ChangeDict[r] == ChangeKindWithDependency.None
where !fileSet.Contains(r)
from i in ldg.GetDependencyReportedBy(r)
select i);
}
}
}
private static void PrepareCachedOutput(
string intermediateFolder,
BuildInfo lastBuildInfo,
string appendContent,
List <ManifestItem> manifestItems,
string additionalFileExtension,
params string[] fileNames)
{
var baseFolder = Path.Combine(intermediateFolder, lastBuildInfo.DirectoryName);
var postProcessOutputs = lastBuildInfo.PostProcessInfo.PostProcessOutputs;
foreach (var fileName in fileNames)
{
var cachedHtmlName = IncrementalUtility.CreateRandomFileName(baseFolder);
var htmlContent = $"{fileName}{appendContent}";
CreateFile($"{cachedHtmlName}", htmlContent, baseFolder);
postProcessOutputs.Add($"{fileName}.html", cachedHtmlName);
var cachedMetaName = IncrementalUtility.CreateRandomFileName(baseFolder);
CreateFile($"{cachedMetaName}", $"{fileName}{MetaAppendContent}", baseFolder);
postProcessOutputs.Add($"{fileName}.mta.json", cachedMetaName);
if (!string.IsNullOrEmpty(additionalFileExtension))
{
var relativePath = $"{fileName}{additionalFileExtension}";
var cachedManifestItemsFileName = IncrementalUtility.CreateRandomFileName(baseFolder);
CreateFile($"{cachedManifestItemsFileName}", htmlContent, baseFolder);
postProcessOutputs.Add(relativePath, cachedManifestItemsFileName);
var item = manifestItems.FirstOrDefault(i => Path.ChangeExtension(i.SourceRelativePath, null) == fileName);
if (item != null)
{
item.OutputFiles.Add($"{additionalFileExtension}", new OutputFileInfo {
RelativePath = relativePath
});
lastBuildInfo.PostProcessInfo.ManifestItems.Add(item);
}
}
}
}
public void SaveIntermediateModel()
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
foreach (var pair in ModelLoadInfo)
{
IncrementalUtility.RetryIO(() =>
{
string fileName = IncrementalUtility.GetRandomEntry(IncrementalBaseDir);
if (pair.Value == LoadPhase.None)
{
if (LastIntermediateModelManifest == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key.FullPath}");
}
string lfn;
if (!LastIntermediateModelManifest.Models.TryGetValue(pair.Key.File, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key.FullPath}");
}
File.Move(Path.Combine(LastIncrementalBaseDir, lfn), Path.Combine(IncrementalBaseDir, fileName));
}
else
{
var key = RelativePath.NormalizedWorkingFolder + pair.Key.File;
var model = Models.Find(m => m.Key == key);
using (var stream = File.Create(Path.Combine(IncrementalBaseDir, fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
}
CurrentIntermediateModelManifest.Models.Add(pair.Key.File, fileName);
});
}
}
public void ReportReference(FileModel currentFileModel, string reference, string referenceType)
{
if (currentFileModel == null)
{
throw new ArgumentNullException(nameof(currentFileModel));
}
if (string.IsNullOrEmpty(reference))
{
throw new ArgumentNullException(nameof(reference));
}
if (referenceType == null)
{
throw new ArgumentNullException(nameof(referenceType));
}
if (DependencyGraph == null)
{
return;
}
string file = IncrementalUtility.GetDependencyKey(currentFileModel.OriginalFileAndType);
DependencyGraph.ReportReference(new ReferenceItem(new DependencyItemSourceInfo(referenceType, reference), file, file));
}
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
foreach (var ofi in result.OutputFiles.Values)
{
if (ofi.LinkToPath != null &&
ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length &&
ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir) &&
(ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/'))
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
}
}
return(result);
}
private void CopyToCurrentCache(List <ManifestItem> increItems)
{
foreach (var item in from mi in increItems
from oi in mi.OutputFiles.Values
where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir)
select oi)
{
string cachedFileName;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to current cache.
var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName);
var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName);
File.Copy(lastCachedFile, currentCachedFile);
item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName);
});
}
}
public void TestIncrementalBasicScenario()
{
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
var lastBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder),
PostProcessInfo = new PostProcessInfo()
};
lastBuildInfo.PostProcessInfo.PostProcessorInfos.Add(new PostProcessorInfo
{
Name = typeof(AppendStringPostProcessor).Name
});
// Exclude c, which is not incremental
var preparedManifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
PrepareCachedOutput(intermediateFolder, lastBuildInfo, AppendStringPostProcessor.AppendString, preparedManifest.Files, AppendStringPostProcessor.AdditionalExtensionString, "a", "b");
var postProcessors = GetPostProcessors(typeof(AppendStringPostProcessor));
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.True(increContext.ShouldTraceIncrementalInfo);
Assert.True(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.Equal(3, manifest.Files.Count);
Assert.True(manifest.Files.Single(i => i.SourceRelativePath == "a.md").IsIncremental);
Assert.True(manifest.Files.Single(i => i.SourceRelativePath == "b.md").IsIncremental);
Assert.False(manifest.Files.Single(i => i.SourceRelativePath == "c.md").IsIncremental);
foreach (var file in manifest.Files)
{
Assert.True(file.OutputFiles.ContainsKey(AppendStringPostProcessor.AdditionalExtensionString));
}
// Check output content
VerifyOutput(outputFolder, AppendStringPostProcessor.AppendString, "a", "b", "c");
// Check cached PostProcessInfo
Assert.NotNull(currentBuildInfo.PostProcessInfo);
var postProcessorInfos = currentBuildInfo.PostProcessInfo.PostProcessorInfos;
Assert.Equal(1, currentBuildInfo.PostProcessInfo.PostProcessorInfos.Count);
Assert.Equal($"{typeof(AppendStringPostProcessor).Name}", postProcessorInfos[0].Name);
Assert.Null(postProcessorInfos[0].IncrementalContextHash);
var postProcessOutputs = currentBuildInfo.PostProcessInfo.PostProcessOutputs;
Assert.Equal(9, postProcessOutputs.Count);
VerifyCachedOutput(Path.Combine(intermediateFolder, currentBuildInfo.DirectoryName), postProcessOutputs, AppendStringPostProcessor.AppendString, AppendStringPostProcessor.AdditionalExtensionString, "a", "b", "c");
// Check incremental info
Assert.Equal(1, manifest.IncrementalInfo.Count);
Assert.Equal(true, manifest.IncrementalInfo[0].Status.CanIncremental);
Assert.Equal(IncrementalPhase.PostProcessing, manifest.IncrementalInfo[0].Status.IncrementalPhase);
Assert.Equal("Can support incremental post processing.", manifest.IncrementalInfo[0].Status.Details);
}
public void Build(IList <DocumentBuildParameters> parameters, string outputDirectory)
{
if (parameters == null)
{
throw new ArgumentNullException(nameof(parameters));
}
if (parameters.Count == 0)
{
throw new ArgumentException("Parameters are empty.", nameof(parameters));
}
var markdownServiceProvider = CompositionContainer.GetExport <IMarkdownServiceProvider>(_container, parameters[0].MarkdownEngineName);
if (markdownServiceProvider == null)
{
Logger.LogError($"Unable to find markdown engine: {parameters[0].MarkdownEngineName}");
throw new DocfxException($"Unable to find markdown engine: {parameters[0].MarkdownEngineName}");
}
Logger.LogInfo($"Markdown engine is {parameters[0].MarkdownEngineName}");
var logCodesLogListener = new LogCodesLogListener();
Logger.RegisterListener(logCodesLogListener);
// Load schema driven processor from template
var sdps = LoadSchemaDrivenDocumentProcessors(parameters[0]).ToList();
if (sdps.Count > 0)
{
Logger.LogInfo($"{sdps.Count()} schema driven document processor plug-in(s) loaded.");
Processors = Processors.Union(sdps);
}
BuildInfo lastBuildInfo = null;
var currentBuildInfo =
new BuildInfo
{
BuildStartTime = DateTime.UtcNow,
DocfxVersion = EnvironmentContext.Version,
};
try
{
lastBuildInfo = BuildInfo.Load(_intermediateFolder, true);
currentBuildInfo.CommitFromSHA = _commitFromSHA;
currentBuildInfo.CommitToSHA = _commitToSHA;
if (_intermediateFolder != null)
{
currentBuildInfo.PluginHash = ComputePluginHash(_assemblyList);
currentBuildInfo.TemplateHash = _templateHash;
if (!_cleanupCacheHistory && lastBuildInfo != null)
{
// Reuse the directory for last incremental if cleanup is disabled
currentBuildInfo.DirectoryName = lastBuildInfo.DirectoryName;
}
else
{
currentBuildInfo.DirectoryName = IncrementalUtility.CreateRandomDirectory(Environment.ExpandEnvironmentVariables(_intermediateFolder));
}
}
_postProcessorsManager.IncrementalInitialize(_intermediateFolder, currentBuildInfo, lastBuildInfo, parameters[0].ForcePostProcess, parameters[0].MaxParallelism);
var manifests = new List <Manifest>();
bool transformDocument = false;
if (parameters.All(p => p.Files.Count == 0))
{
Logger.LogWarning(
$"No file found, nothing will be generated. Please make sure docfx.json is correctly configured.",
code: WarningCodes.Build.EmptyInputFiles);
}
var noContentFound = true;
var emptyContentGroups = new List <string>();
foreach (var parameter in parameters)
{
if (parameter.CustomLinkResolver != null)
{
if (_container.TryGetExport(parameter.CustomLinkResolver, out ICustomHrefGenerator chg))
{
parameter.ApplyTemplateSettings.HrefGenerator = chg;
}
else
{
Logger.LogWarning($"Custom href generator({parameter.CustomLinkResolver}) is not found.");
}
}
FileAbstractLayerBuilder falBuilder;
if (_intermediateFolder == null)
{
falBuilder = FileAbstractLayerBuilder.Default
.ReadFromRealFileSystem(EnvironmentContext.BaseDirectory)
.WriteToRealFileSystem(parameter.OutputBaseDir);
}
else
{
falBuilder = FileAbstractLayerBuilder.Default
.ReadFromRealFileSystem(EnvironmentContext.BaseDirectory)
.WriteToLink(Path.Combine(_intermediateFolder, currentBuildInfo.DirectoryName));
}
if (!string.IsNullOrEmpty(parameter.FALName))
{
if (_container.TryGetExport <IInputFileAbstractLayerBuilderProvider>(
parameter.FALName, out var provider))
{
falBuilder = provider.Create(falBuilder, parameter);
}
else
{
Logger.LogWarning($"Input fal builder provider not found, name: {parameter.FALName}.");
}
}
EnvironmentContext.FileAbstractLayerImpl = falBuilder.Create();
if (parameter.ApplyTemplateSettings.TransformDocument)
{
transformDocument = true;
}
if (parameter.Files.Count == 0)
{
manifests.Add(new Manifest());
}
else
{
if (!parameter.Files.EnumerateFiles().Any(s => s.Type == DocumentType.Article))
{
if (!string.IsNullOrEmpty(parameter.GroupInfo?.Name))
{
emptyContentGroups.Add(parameter.GroupInfo.Name);
}
}
else
{
noContentFound = false;
}
parameter.Metadata = _postProcessorsManager.PrepareMetadata(parameter.Metadata);
if (!string.IsNullOrEmpty(parameter.VersionName))
{
Logger.LogInfo($"Start building for version: {parameter.VersionName}");
}
using (new LoggerPhaseScope("BuildCore"))
{
manifests.Add(BuildCore(parameter, markdownServiceProvider, currentBuildInfo, lastBuildInfo));
}
}
}
if (noContentFound)
{
Logger.LogWarning(
$"No content file found. Please make sure the content section of docfx.json is correctly configured.",
code: WarningCodes.Build.EmptyInputContents);
}
else if (emptyContentGroups.Count > 0)
{
Logger.LogWarning(
$"No content file found in group: {string.Join(",", emptyContentGroups)}. Please make sure the content section of docfx.json is correctly configured.",
code: WarningCodes.Build.EmptyInputContents);
}
using (new LoggerPhaseScope("Postprocess", LogLevel.Verbose))
{
var generatedManifest = ManifestUtility.MergeManifest(manifests);
generatedManifest.SitemapOptions = parameters.FirstOrDefault()?.SitemapOptions;
ManifestUtility.RemoveDuplicateOutputFiles(generatedManifest.Files);
ManifestUtility.ApplyLogCodes(generatedManifest.Files, logCodesLogListener.Codes);
EnvironmentContext.FileAbstractLayerImpl =
FileAbstractLayerBuilder.Default
.ReadFromManifest(generatedManifest, parameters[0].OutputBaseDir)
.WriteToManifest(generatedManifest, parameters[0].OutputBaseDir)
.Create();
using (new PerformanceScope("Process"))
{
_postProcessorsManager.Process(generatedManifest, outputDirectory);
}
using (new PerformanceScope("Dereference"))
{
if (parameters[0].KeepFileLink)
{
var count = (from f in generatedManifest.Files
from o in f.OutputFiles
select o.Value into v
where v.LinkToPath != null
select v).Count();
if (count > 0)
{
Logger.LogInfo($"Skip dereferencing {count} files.");
}
}
else
{
generatedManifest.Dereference(parameters[0].OutputBaseDir, parameters[0].MaxParallelism);
}
}
using (new PerformanceScope("SaveManifest"))
{
// Save to manifest.json
EnvironmentContext.FileAbstractLayerImpl =
FileAbstractLayerBuilder.Default
.ReadFromRealFileSystem(parameters[0].OutputBaseDir)
.WriteToRealFileSystem(parameters[0].OutputBaseDir)
.Create();
SaveManifest(generatedManifest);
}
using (new PerformanceScope("Cleanup"))
{
EnvironmentContext.FileAbstractLayerImpl = null;
// overwrite intermediate cache files
if (_intermediateFolder != null && transformDocument)
{
try
{
currentBuildInfo.IsValid = Logger.WarningCount < Logger.WarningThrottling;
currentBuildInfo.Save(_intermediateFolder);
if (_cleanupCacheHistory)
{
ClearCacheExcept(currentBuildInfo.DirectoryName);
}
}
catch (Exception ex)
{
Logger.LogWarning($"Error happened while saving cache. Message: {ex.Message}.");
}
}
}
}
}
catch
{
// Leave cache folder there as it contains historical data
// exceptions happens in this build does not corrupt the cache theoretically
// however the cache file created by this build will never be cleaned up with DisableIncrementalFolderCleanup option
if (_intermediateFolder != null && _cleanupCacheHistory)
{
ClearCacheExcept(lastBuildInfo?.DirectoryName);
}
throw;
}
finally
{
Logger.UnregisterListener(logCodesLogListener);
}
}
public void TestIncrementalWithContextChange()
{
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
var lastBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder),
PostProcessInfo = new PostProcessInfo()
};
lastBuildInfo.PostProcessInfo.PostProcessorInfos.Add(new PostProcessorInfo
{
Name = typeof(AppendIntegerPostProcessor).Name
});
// Add post processor which has changed context hash
var postProcessors = GetPostProcessors(typeof(AppendIntegerPostProcessor));
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.True(increContext.ShouldTraceIncrementalInfo);
Assert.False(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.True(manifest.Files.All(f => f.IsIncremental == false));
// Check output content
VerifyOutput(outputFolder, AppendIntegerPostProcessor.AppendInteger, "a", "b", "c");
// Check cached PostProcessInfo
Assert.NotNull(currentBuildInfo.PostProcessInfo);
var postProcessorInfos = currentBuildInfo.PostProcessInfo.PostProcessorInfos;
Assert.Equal(1, currentBuildInfo.PostProcessInfo.PostProcessorInfos.Count);
Assert.Equal($"{typeof(AppendIntegerPostProcessor).Name}", postProcessorInfos[0].Name);
Assert.NotNull(postProcessorInfos[0].IncrementalContextHash);
var postProcessOutputs = currentBuildInfo.PostProcessInfo.PostProcessOutputs;
Assert.Equal(6, postProcessOutputs.Count);
VerifyCachedOutput(Path.Combine(intermediateFolder, currentBuildInfo.DirectoryName), postProcessOutputs, AppendIntegerPostProcessor.AppendInteger, null, "a", "b", "c");
Assert.Equal <ManifestItem>(manifest.Files, currentBuildInfo.PostProcessInfo.ManifestItems);
// Check incremental info
Assert.Equal(1, manifest.IncrementalInfo.Count);
Assert.Equal(false, manifest.IncrementalInfo[0].Status.CanIncremental);
Assert.Equal(IncrementalPhase.PostProcessing, manifest.IncrementalInfo[0].Status.IncrementalPhase);
Assert.Equal(@"Cannot support incremental post processing, the reason is: post processor info changed from last {""Name"":""AppendIntegerPostProcessor""} to current {""Name"":""AppendIntegerPostProcessor"",""IncrementalContextHash"":""1024""}.",
manifest.IncrementalInfo[0].Status.Details);
}
public void TestIncrementalWithFileInDirectory()
{
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
var lastBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder),
PostProcessInfo = new PostProcessInfo()
};
lastBuildInfo.PostProcessInfo.PostProcessorInfos.Add(new PostProcessorInfo
{
Name = typeof(AppendStringPostProcessor).Name
});
// Exclude c, which is not incremental
var preparedManifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental_with_directory.json");
PrepareCachedOutput(intermediateFolder, lastBuildInfo, AppendStringPostProcessor.AppendString, preparedManifest.Files, AppendStringPostProcessor.AdditionalExtensionString, "a/b");
var postProcessors = GetPostProcessors(typeof(AppendStringPostProcessor));
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.True(increContext.ShouldTraceIncrementalInfo);
Assert.True(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental_with_directory.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a/b", "c");
CreateFile("breadcrumb.json", "breadcrumb", outputFolder);
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.Equal(3, manifest.Files.Count);
Assert.True(manifest.Files.Single(i => i.SourceRelativePath == "a/b.md").IsIncremental);
Assert.False(manifest.Files.Single(i => i.SourceRelativePath == "c.md").IsIncremental);
Assert.False(manifest.Files.Single(i => i.SourceRelativePath == "breadcrumb.json").IsIncremental);
// Check output content
VerifyOutput(outputFolder, AppendStringPostProcessor.AppendString, "a/b", "c");
Assert.Equal("breadcrumb", EnvironmentContext.FileAbstractLayer.ReadAllText(Path.Combine(outputFolder, "breadcrumb.json")));
// Check cached PostProcessInfo
Assert.NotNull(currentBuildInfo.PostProcessInfo);
var postProcessorInfos = currentBuildInfo.PostProcessInfo.PostProcessorInfos;
Assert.Equal(1, currentBuildInfo.PostProcessInfo.PostProcessorInfos.Count);
Assert.Equal($"{typeof(AppendStringPostProcessor).Name}", postProcessorInfos[0].Name);
Assert.Null(postProcessorInfos[0].IncrementalContextHash);
Assert.Equal <ManifestItem>(manifest.Files, currentBuildInfo.PostProcessInfo.ManifestItems);
var postProcessOutputs = currentBuildInfo.PostProcessInfo.PostProcessOutputs;
Assert.Equal(6, postProcessOutputs.Count);
VerifyCachedOutput(Path.Combine(intermediateFolder, currentBuildInfo.DirectoryName), postProcessOutputs, AppendStringPostProcessor.AppendString, AppendStringPostProcessor.AdditionalExtensionString, "a/b", "c");
}
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext)
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
var mi = incrementalContext.GetModelLoadInfo(this);
var lmm = incrementalContext.GetLastIntermediateModelManifest(this);
var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this);
Parallel.ForEach(mi, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
}, pair =>
{
IncrementalUtility.RetryIO(() =>
{
var items = new List <ModelManifestItem>();
if (pair.Value == null)
{
if (lmm == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}");
}
if (!lmm.Models.TryGetValue(pair.Key, out List <ModelManifestItem> lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}");
}
if (FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals(
incrementalContext.BaseDir,
incrementalContext.LastBaseDir))
{
items.AddRange(lfn);
}
else
{
foreach (var item in lfn)
{
// use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted.
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
File.Copy(
Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.LastBaseDir), item.FilePath),
Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir), fileName));
items.Add(new ModelManifestItem()
{
SourceFilePath = item.SourceFilePath, FilePath = fileName
});
}
}
}
else
{
var models = Models.Where(m => m.OriginalFileAndType.File == pair.Key).ToList();
foreach (var model in models)
{
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
using (var stream = File.Create(
Path.Combine(
Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir),
fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
items.Add(new ModelManifestItem()
{
SourceFilePath = model.FileAndType.File, FilePath = fileName
});
}
}
lock (cmm)
{
cmm.Models.Add(pair.Key, items);
}
});
});
}
public void TestIncrementalWithFirstCanIncrementalButSecondShouldnotTraceIncrementalInfo()
{
// | Should trace incremental info | Can incremental |
// ---------------------------------------------------------
// First | yes | yes |
// Second | no | no |
// Step 1: trace intermediate info and post process incrementally
var intermediateFolder = GetRandomFolder();
var currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
var lastBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder),
PostProcessInfo = new PostProcessInfo()
};
lastBuildInfo.PostProcessInfo.PostProcessorInfos.Add(new PostProcessorInfo
{
Name = typeof(AppendStringPostProcessor).Name
});
// Exclude c, which is not incremental
var preparedManifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
PrepareCachedOutput(intermediateFolder, lastBuildInfo, AppendStringPostProcessor.AppendString, preparedManifest.Files, AppendStringPostProcessor.AdditionalExtensionString, "a", "b");
var postProcessors = GetPostProcessors(typeof(AppendStringPostProcessor));
var appendString = $"{AppendStringPostProcessor.AppendString}";
var increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.True(increContext.ShouldTraceIncrementalInfo);
Assert.True(increContext.IsIncremental);
var increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
var manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
var outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.Equal(3, manifest.Files.Count);
Assert.True(manifest.Files.Single(i => i.SourceRelativePath == "a.md").IsIncremental);
Assert.True(manifest.Files.Single(i => i.SourceRelativePath == "b.md").IsIncremental);
Assert.False(manifest.Files.Single(i => i.SourceRelativePath == "c.md").IsIncremental);
foreach (var file in manifest.Files)
{
Assert.True(file.OutputFiles.ContainsKey(AppendStringPostProcessor.AdditionalExtensionString));
}
// Check output content
VerifyOutput(outputFolder, appendString, "a", "b", "c");
// Check cached PostProcessInfo
Assert.NotNull(currentBuildInfo.PostProcessInfo);
var postProcessorInfos = currentBuildInfo.PostProcessInfo.PostProcessorInfos;
Assert.Equal(1, currentBuildInfo.PostProcessInfo.PostProcessorInfos.Count);
Assert.Equal($"{typeof(AppendStringPostProcessor).Name}", postProcessorInfos[0].Name);
Assert.Null(postProcessorInfos[0].IncrementalContextHash);
Assert.Equal <ManifestItem>(manifest.Files, currentBuildInfo.PostProcessInfo.ManifestItems);
var postProcessOutputs = currentBuildInfo.PostProcessInfo.PostProcessOutputs;
Assert.Equal(9, postProcessOutputs.Count);
VerifyCachedOutput(Path.Combine(intermediateFolder, currentBuildInfo.DirectoryName), postProcessOutputs, appendString, AppendStringPostProcessor.AdditionalExtensionString, "a", "b", "c");
// Step 2: should not trace inter incremental post process
currentBuildInfo = new BuildInfo
{
DirectoryName = IncrementalUtility.CreateRandomDirectory(intermediateFolder)
};
lastBuildInfo = new BuildInfo
{
DirectoryName = Path.GetFileName(increContext.CurrentBaseDir),
PostProcessInfo = increContext.CurrentInfo
};
// Add post processor which not supports incremental
postProcessors.AddRange(GetPostProcessors(typeof(NonIncrementalPostProcessor)));
increContext = new IncrementalPostProcessorsContext(intermediateFolder, currentBuildInfo, lastBuildInfo, postProcessors, true);
// Check context
Assert.False(increContext.ShouldTraceIncrementalInfo);
Assert.False(increContext.IsIncremental);
increPostProcessorHandler = new PostProcessorsHandlerWithIncremental(PostProcessorsHandler, increContext);
manifest = JsonUtility.Deserialize <Manifest>("PostProcessors/Data/manifest_incremental.json");
outputFolder = GetRandomFolder();
PrepareOutput(outputFolder, "a", "b", "c");
increPostProcessorHandler.Handle(postProcessors, manifest, outputFolder);
// Check incremental flag
Assert.True(manifest.Files.All(f => f.IsIncremental == false));
// Check output content
VerifyOutput(outputFolder, appendString, "a", "b", "c");
// Check cached PostProcessInfo should be null
Assert.Null(currentBuildInfo.PostProcessInfo);
}
