private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath) { var result = item.Clone(); result.IsIncremental = true; result.SourceRelativePath = sourceRelativePath; // Copy when current base dir is not last base dir if (!FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals( IncrementalContext.BaseDir, IncrementalContext.LastBaseDir)) { foreach (var ofi in result.OutputFiles.Values) { if (ofi.LinkToPath != null && ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length && ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir, StringComparison.Ordinal) && (ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/')) { IncrementalUtility.RetryIO(() => { var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir)); File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path)); ofi.LinkToPath = path; }); } } } return(result); }
private void CopyToCurrentCache(List <ManifestItem> increItems) { using (new LoggerPhaseScope("CopyToCurrentCache", LogLevel.Verbose)) { var itemsToBeCopied = from mi in increItems from oi in mi.OutputFiles.Values where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir) select oi; Parallel.ForEach( itemsToBeCopied, new ParallelOptions { MaxDegreeOfParallelism = _increContext.MaxParallelism }, item => { string cachedFileName; if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName)) { throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}."); } IncrementalUtility.RetryIO(() => { // Copy last cached file to current cache. var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir); var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName); var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName); File.Copy(lastCachedFile, currentCachedFile); item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName); }); }); } }
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath) { var result = item.Clone(); result.IsIncremental = true; result.SourceRelativePath = sourceRelativePath; Parallel.ForEach( from ofi in result.OutputFiles.Values where ofi.LinkToPath != null where ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length where ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir) where (ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/') select ofi, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, ofi => { IncrementalUtility.RetryIO(() => { var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir)); File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path)); ofi.LinkToPath = path; }); }); return(result); }
private void TraceIncremental(List <ManifestItem> increItems) { foreach (var outputRelPath in GetOutputRelativePaths(increItems)) { string lastCachedRelPath; if (_increContext.LastInfo == null) { throw new BuildCacheException("Last incremental post processor info should not be null."); } if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath)) { throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}."); } IncrementalUtility.RetryIO(() => { var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath); var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir); // Copy last cached file to current cached file EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName)); _increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName); }); } }
private void SaveOutputs(IEnumerable <HostService> hostServices) { var outputDir = Context.BuildOutputFolder; var lo = LastBuildVersionInfo?.BuildOutputs; var outputItems = (from m in Context.ManifestItems from output in m.OutputFiles.Values select new { Path = output.RelativePath, SourcePath = m.SourceRelativePath, } into items group items by items.SourcePath).ToDictionary(g => g.Key, g => g.Select(p => p.Path).ToList(), FilePathComparer.OSPlatformSensitiveStringComparer); foreach (var h in hostServices.Where(h => h.ShouldTraceIncrementalInfo)) { foreach (var pair in IncrementalContext.GetModelLoadInfo(h)) { List <string> items; if (!outputItems.TryGetValue(pair.Key, out items)) { continue; } foreach (var path in items) { // path might be duplicate. for example, files with same name in different input folders are mapped to same output folder. if (CurrentBuildVersionInfo.BuildOutputs.ContainsKey(path)) { continue; } string fileName = IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir); string fullPath = Path.Combine(outputDir, path); IncrementalUtility.RetryIO(() => { if (pair.Value == null) { if (lo == null) { throw new BuildCacheException($"Full build hasn't loaded build outputs."); } string lfn; if (!lo.TryGetValue(path, out lfn)) { throw new BuildCacheException($"Last build hasn't loaded output: {path}."); } Directory.CreateDirectory(Path.GetDirectoryName(fullPath)); File.Copy(Path.Combine(IncrementalContext.LastBaseDir, lfn), fullPath, true); } File.Copy(fullPath, Path.Combine(IncrementalContext.BaseDir, fileName)); CurrentBuildVersionInfo.BuildOutputs[path] = fileName; }); } } } }
private void TraceNoneIncremental(string outputFolder, List <ManifestItem> nonIncreItems) { foreach (var outputRelPath in GetOutputRelativePaths(nonIncreItems, ExcludeType)) { IncrementalUtility.RetryIO(() => { var outputPath = Path.Combine(outputFolder, outputRelPath); var currentCachedFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir); // Copy output to current cached file EnvironmentContext.FileAbstractLayer.Copy(outputPath, Path.Combine(_increContext.CurrentBaseDir, currentCachedFileName)); _increContext.CurrentInfo.PostProcessOutputs.Add(outputRelPath, currentCachedFileName); }); } }
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext) { if (!ShouldTraceIncrementalInfo) { return; } var processor = (ISupportIncrementalDocumentProcessor)Processor; var mi = incrementalContext.GetModelLoadInfo(this); var lmm = incrementalContext.GetLastIntermediateModelManifest(this); var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this); foreach (var pair in mi) { IncrementalUtility.RetryIO(() => { string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir); if (pair.Value == null) { if (lmm == null) { throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}"); } string lfn; if (!lmm.Models.TryGetValue(pair.Key, out lfn)) { throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}"); } // use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted. File.Copy(Path.Combine(incrementalContext.LastBaseDir, lfn), Path.Combine(incrementalContext.BaseDir, fileName)); } else { var key = RelativePath.NormalizedWorkingFolder + pair.Key; var model = Models.Find(m => m.Key == key); using (var stream = File.Create(Path.Combine(incrementalContext.BaseDir, fileName))) { processor.SaveIntermediateModel(model, stream); } } cmm.Models.Add(pair.Key, fileName); }); } }
private void CopyToOutput(List <ManifestItem> increItems, string outputFolder) { foreach (var outputRelPath in GetOutputRelativePaths(increItems)) { string lastCachedRelPath; if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(outputRelPath, out lastCachedRelPath)) { throw new BuildCacheException($"Last incremental post processor outputs should contain {outputRelPath}."); } IncrementalUtility.RetryIO(() => { // Copy last cached file to output var outputPath = Path.Combine(outputFolder, outputRelPath); var lastCachedFile = Path.Combine(_increContext.LastBaseDir, lastCachedRelPath); EnvironmentContext.FileAbstractLayer.Copy(lastCachedFile, outputPath); }); } }
public void SaveIntermediateModel() { if (!ShouldTraceIncrementalInfo) { return; } var processor = (ISupportIncrementalDocumentProcessor)Processor; foreach (var pair in ModelLoadInfo) { IncrementalUtility.RetryIO(() => { string fileName = IncrementalUtility.GetRandomEntry(IncrementalBaseDir); if (pair.Value == LoadPhase.None) { if (LastIntermediateModelManifest == null) { throw new BuildCacheException($"Full build hasn't loaded model {pair.Key.FullPath}"); } string lfn; if (!LastIntermediateModelManifest.Models.TryGetValue(pair.Key.File, out lfn)) { throw new BuildCacheException($"Last build hasn't loaded model {pair.Key.FullPath}"); } File.Move(Path.Combine(LastIncrementalBaseDir, lfn), Path.Combine(IncrementalBaseDir, fileName)); } else { var key = RelativePath.NormalizedWorkingFolder + pair.Key.File; var model = Models.Find(m => m.Key == key); using (var stream = File.Create(Path.Combine(IncrementalBaseDir, fileName))) { processor.SaveIntermediateModel(model, stream); } } CurrentIntermediateModelManifest.Models.Add(pair.Key.File, fileName); }); } }
private ManifestItem UpdateItem(ManifestItem item, string sourceRelativePath) { var result = item.Clone(); result.IsIncremental = true; result.SourceRelativePath = sourceRelativePath; foreach (var ofi in result.OutputFiles.Values) { if (ofi.LinkToPath != null && ofi.LinkToPath.Length > IncrementalContext.LastBaseDir.Length && ofi.LinkToPath.StartsWith(IncrementalContext.LastBaseDir) && (ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '\\' || ofi.LinkToPath[IncrementalContext.LastBaseDir.Length] == '/')) { IncrementalUtility.RetryIO(() => { var path = Path.Combine(IncrementalContext.BaseDir, IncrementalUtility.GetRandomEntry(IncrementalContext.BaseDir)); File.Copy(Environment.ExpandEnvironmentVariables(ofi.LinkToPath), Environment.ExpandEnvironmentVariables(path)); ofi.LinkToPath = path; }); } } return(result); }
private void CopyToCurrentCache(List <ManifestItem> increItems) { foreach (var item in from mi in increItems from oi in mi.OutputFiles.Values where oi.LinkToPath != null && oi.LinkToPath.StartsWith(_increContext.LastBaseDir) select oi) { string cachedFileName; if (!_increContext.LastInfo.PostProcessOutputs.TryGetValue(item.RelativePath, out cachedFileName)) { throw new BuildCacheException($"Last incremental post processor outputs should contain {item.RelativePath}."); } IncrementalUtility.RetryIO(() => { // Copy last cached file to current cache. var newFileName = IncrementalUtility.GetRandomEntry(_increContext.CurrentBaseDir); var currentCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.CurrentBaseDir), newFileName); var lastCachedFile = Path.Combine(Environment.ExpandEnvironmentVariables(_increContext.LastBaseDir), cachedFileName); File.Copy(lastCachedFile, currentCachedFile); item.LinkToPath = Path.Combine(_increContext.CurrentBaseDir, newFileName); }); } }
public void SaveIntermediateModel(IncrementalBuildContext incrementalContext) { if (!ShouldTraceIncrementalInfo) { return; } var processor = (ISupportIncrementalDocumentProcessor)Processor; var mi = incrementalContext.GetModelLoadInfo(this); var lmm = incrementalContext.GetLastIntermediateModelManifest(this); var cmm = incrementalContext.GetCurrentIntermediateModelManifest(this); Parallel.ForEach(mi, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, pair => { IncrementalUtility.RetryIO(() => { var items = new List <ModelManifestItem>(); if (pair.Value == null) { if (lmm == null) { throw new BuildCacheException($"Full build hasn't loaded model {pair.Key}"); } if (!lmm.Models.TryGetValue(pair.Key, out List <ModelManifestItem> lfn)) { throw new BuildCacheException($"Last build hasn't loaded model {pair.Key}"); } if (FilePathComparerWithEnvironmentVariable.OSPlatformSensitiveRelativePathComparer.Equals( incrementalContext.BaseDir, incrementalContext.LastBaseDir)) { items.AddRange(lfn); } else { foreach (var item in lfn) { // use copy rather than move because if the build failed, the intermediate files of last successful build shouldn't be corrupted. string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir); File.Copy( Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.LastBaseDir), item.FilePath), Path.Combine(Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir), fileName)); items.Add(new ModelManifestItem() { SourceFilePath = item.SourceFilePath, FilePath = fileName }); } } } else { var models = Models.Where(m => m.OriginalFileAndType.File == pair.Key).ToList(); foreach (var model in models) { string fileName = IncrementalUtility.GetRandomEntry(incrementalContext.BaseDir); using (var stream = File.Create( Path.Combine( Environment.ExpandEnvironmentVariables(incrementalContext.BaseDir), fileName))) { processor.SaveIntermediateModel(model, stream); } items.Add(new ModelManifestItem() { SourceFilePath = model.FileAndType.File, FilePath = fileName }); } } lock (cmm) { cmm.Models.Add(pair.Key, items); } }); }); }