private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
// Copy when current base dir is not last base dir
if (!FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals(
IncrementalContext.BaseDir,
IncrementalContext.LastBaseDir))
{
foreach (var ofi in result.OutputFiles.Values)
{
if (ofi.LinkToPath != null &&
ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length &&
ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir, StringComparison.Ordinal) &&
(ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/'))
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
}
}
}
return(result);
}
private void CopyToCurrentCache(List <ManifestItem> increItems)
{
using (new LoggerPhaseScope("CopyToCurrentCache", LogLevel.Verbose))
{
var itemsToBeCopied = from mi in increItems
from oi in mi.OutputFiles.Values
where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir)
select oi;
Parallel.ForEach(
itemsToBeCopied,
new ParallelOptions {
MaxDegreeOfParallelism = _increContext.MaxParallelism
},
item =>
{
string cachedFileName;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to current cache.
var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName);
var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName);
File.Copy(lastCachedFile, currentCachedFile);
item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName);
});
});
}
}
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
Parallel.ForEach(
from ofi in result.OutputFiles.Values
where ofi.LinkToPath != null
where ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length
where ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir)
where (ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/')
select ofi,
new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
},
ofi =>
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
});
return(result);
}
private void TraceIncremental(List <ManifestItem> increItems)
{
foreach (var outputRelPath in GetOutputRelativePaths(increItems))
{
string lastCachedRelPath;
if (_increContext.LastInfo == null)
{
throw new BuildCacheException("Last incremental post processor info should not be null.");
}
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}.");
}
IncrementalUtility.RetryIO(() =>
{
var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath);
var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
// Copy last cached file to current cached file
EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName));
_increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName);
});
}
}
private void SaveOutputs(IEnumerable <HostService> hostServices)
{
var outputDir = Context.BuildOutputFolder;
var lo = LastBuildVersionInfo?.BuildOutputs;
var outputItems = (from m in Context.ManifestItems
from output in m.OutputFiles.Values
select new
{
Path = output.RelativePath,
SourcePath = m.SourceRelativePath,
} into items
group items by items.SourcePath).ToDictionary(g => g.Key, g => g.Select(p => p.Path).ToList(), FilePathComparer.OSPlatformSensitiveStringComparer);
foreach (var h in hostServices.Where(h => h.ShouldTraceIncrementalInfo))
{
foreach (var pair in IncrementalContext.GetModelLoadInfo(h))
{
List <string> items;
if (!outputItems.TryGetValue(pair.Key, out items))
{
continue;
}
foreach (var path in items)
{
// path might be duplicate. for example, files with same name in different input folders are mapped to same output folder.
if (CurrentBuildVersionInfo.BuildOutputs.ContainsKey(path))
{
continue;
}
string fileName = IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir);
string fullPath = Path.Combine(outputDir, path);
IncrementalUtility.RetryIO(() =>
{
if (pair.Value == null)
{
if (lo == null)
{
throw new BuildCacheException($"Full build hasn't loaded build outputs.");
}
string lfn;
if (!lo.TryGetValue(path, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded output: {path}.");
}
Directory.CreateDirectory(Path.GetDirectoryName(fullPath));
File.Copy(Path.Combine(IncrementalContext.LastBaseDir, lfn), fullPath, true);
}
File.Copy(fullPath, Path.Combine(IncrementalContext.BaseDir, fileName));
CurrentBuildVersionInfo.BuildOutputs[path] = fileName;
});
}
}
}
}
private void TraceNoneIncremental(string outputFolder, List <ManifestItem> nonIncreItems)
{
foreach (var outputRelPath in GetOutputRelativePaths(nonIncreItems, ExcludeType))
{
IncrementalUtility.RetryIO(() =>
{
var outputPath = Path.Combine(outputFolder, outputRelPath);
var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
// Copy output to current cached file
EnvironmentContext.FileAbstractLayer.Copy(outputPath, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName));
_increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName);
});
}
}
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext)
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
var mi = incrementalContext.GetModelLoadInfo(this);
var lmm = incrementalContext.GetLastIntermediateModelManifest(this);
var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this);
foreach (var pair in mi)
{
IncrementalUtility.RetryIO(() =>
{
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
if (pair.Value == null)
{
if (lmm == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}");
}
string lfn;
if (!lmm.Models.TryGetValue(pair.Key, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}");
}
// use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted.
File.Copy(Path.Combine(incrementalContext.LastBaseDir, lfn), Path.Combine(incrementalContext.BaseDir, fileName));
}
else
{
var key = RelativePath.NormalizedWorkingFolder + pair.Key;
var model = Models.Find(m => m.Key == key);
using (var stream = File.Create(Path.Combine(incrementalContext.BaseDir, fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
}
cmm.Models.Add(pair.Key, fileName);
});
}
}
private void CopyToOutput(List <ManifestItem> increItems, string outputFolder)
{
foreach (var outputRelPath in GetOutputRelativePaths(increItems))
{
string lastCachedRelPath;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to output
var outputPath = Path.Combine(outputFolder, outputRelPath);
var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath);
EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, outputPath);
});
}
}
public void SaveIntermediateModel()
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
foreach (var pair in ModelLoadInfo)
{
IncrementalUtility.RetryIO(() =>
{
string fileName = IncrementalUtility.GetRandomEntry(IncrementalBaseDir);
if (pair.Value == LoadPhase.None)
{
if (LastIntermediateModelManifest == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key.FullPath}");
}
string lfn;
if (!LastIntermediateModelManifest.Models.TryGetValue(pair.Key.File, out lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key.FullPath}");
}
File.Move(Path.Combine(LastIncrementalBaseDir, lfn), Path.Combine(IncrementalBaseDir, fileName));
}
else
{
var key = RelativePath.NormalizedWorkingFolder + pair.Key.File;
var model = Models.Find(m => m.Key == key);
using (var stream = File.Create(Path.Combine(IncrementalBaseDir, fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
}
CurrentIntermediateModelManifest.Models.Add(pair.Key.File, fileName);
});
}
}
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath)
{
var result = item.Clone();
result.IsIncremental = true;
result.SourceRelativePath = sourceRelativePath;
foreach (var ofi in result.OutputFiles.Values)
{
if (ofi.LinkToPath != null &&
ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length &&
ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir) &&
(ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/'))
{
IncrementalUtility.RetryIO(() =>
{
var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir));
File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path));
ofi.LinkToPath = path;
});
}
}
return(result);
}
private void CopyToCurrentCache(List <ManifestItem> increItems)
{
foreach (var item in from mi in increItems
from oi in mi.OutputFiles.Values
where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir)
select oi)
{
string cachedFileName;
if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName))
{
throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}.");
}
IncrementalUtility.RetryIO(() =>
{
// Copy last cached file to current cache.
var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir);
var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName);
var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName);
File.Copy(lastCachedFile, currentCachedFile);
item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName);
});
}
}
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext)
{
if (!ShouldTraceIncrementalInfo)
{
return;
}
var processor = (ISupportIncrementalDocumentProcessor)Processor;
var mi = incrementalContext.GetModelLoadInfo(this);
var lmm = incrementalContext.GetLastIntermediateModelManifest(this);
var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this);
Parallel.ForEach(mi, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
}, pair =>
{
IncrementalUtility.RetryIO(() =>
{
var items = new List <ModelManifestItem>();
if (pair.Value == null)
{
if (lmm == null)
{
throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}");
}
if (!lmm.Models.TryGetValue(pair.Key, out List <ModelManifestItem> lfn))
{
throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}");
}
if (FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals(
incrementalContext.BaseDir,
incrementalContext.LastBaseDir))
{
items.AddRange(lfn);
}
else
{
foreach (var item in lfn)
{
// use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted.
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
File.Copy(
Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.LastBaseDir), item.FilePath),
Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir), fileName));
items.Add(new ModelManifestItem()
{
SourceFilePath = item.SourceFilePath, FilePath = fileName
});
}
}
}
else
{
var models = Models.Where(m => m.OriginalFileAndType.File == pair.Key).ToList();
foreach (var model in models)
{
string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir);
using (var stream = File.Create(
Path.Combine(
Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir),
fileName)))
{
processor.SaveIntermediateModel(model, stream);
}
items.Add(new ModelManifestItem()
{
SourceFilePath = model.FileAndType.File, FilePath = fileName
});
}
}
lock (cmm)
{
cmm.Models.Add(pair.Key, items);
}
});
});
}
