/// <summary> /// </summary> public void UnregisterManifest(Guid ManifestId) { BuildManifest Manifest = GetManifestById(ManifestId); if (Manifest == null) { return; } string FilePath = Path.Combine(RootPath, ManifestId + ".manifest"); Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unregistering manifest: {0}", FilePath); try { File.Delete(FilePath); string MetaFile = FilePath + ".metadata"; if (File.Exists(MetaFile)) { Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unregistering manifest metadata: {0}", MetaFile); File.Delete(MetaFile); } } catch (Exception Ex) { Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to delete unregistered manifest file {0} with error: {1}", FilePath, Ex.Message); } ManifestFileSystem.RemoveNode(Manifest.VirtualPath); Manifests.Remove(Manifest); }
/// <summary> /// </summary> /// <param name="Manifest"></param> public void AddManifest(BuildManifest Manifest) { ManifestFileSystem.InsertNode(Manifest.VirtualPath, DateTime.UtcNow, Manifest); Manifests.Add(Manifest); PruneManifests(); }
/// <summary> /// </summary> public void PruneUnseenManifests(int MaxDaysOld) { lock (ManifestLastSeenTimes) { for (int i = 0; i < Manifests.Count; i++) { BuildManifest Manifest = Manifests[i]; string Id = Manifest.Guid.ToString(); if (ManifestLastSeenTimes.ContainsKey(Id)) { TimeSpan Elapsed = DateTime.UtcNow - ManifestLastSeenTimes[Id]; if (Elapsed.TotalDays > MaxDaysOld) { Logger.Log(LogLevel.Info, LogCategory.Manifest, "Pruning manifest '{0}' '{1}' as it's {2} days since it was last seen on any peer.", Manifest.Guid.ToString(), Manifest.VirtualPath, (int)Elapsed.TotalDays); UnregisterManifest(Manifest.Guid); // ManifestFileSystem.RemoveNode(Manifest.VirtualPath); //Manifests.RemoveAt(i); i--; ManifestLastSeenTimesDirty = true; } } else { // Put an initial timestamp in for this manifest. MarkAsSeen(Manifest.Guid); ManifestLastSeenTimesDirty = true; } } } }
/// <summary> /// /// </summary> /// <param name="Manifest"></param> /// <param name="Tag"></param> public void UntagManifest(Guid ManifestId, Guid TagId) { BuildManifest Manifest = GetManifestById(ManifestId); if (Manifest == null) { return; } Tag Tag = TagRegistry.GetTagById(TagId); if (Tag == null) { return; } if (Manifest.Metadata != null && Manifest.Metadata.TagIds.Contains(TagId)) { Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unagging manifest {0} with {1}", ManifestId.ToString(), Tag.Name); Manifest.Metadata.ModifiedTime = DateTime.UtcNow; Manifest.Metadata.TagIds.Remove(TagId); StoreMetadata(Manifest); } }
/// <summary> /// /// </summary> /// <param name="Manifest"></param> /// <param name="Tag"></param> public void TagManifest(Guid ManifestId, Guid TagId) { BuildManifest Manifest = GetManifestById(ManifestId); if (Manifest == null) { return; } Tag Tag = TagRegistry.GetTagById(TagId); if (Tag == null) { return; } if (Manifest.Metadata != null && !Manifest.Metadata.TagIds.Contains(TagId)) { Logger.Log(LogLevel.Info, LogCategory.Manifest, "Tagging manifest {0} with {1}", ManifestId.ToString(), Tag.Name); Manifest.Metadata.ModifiedTime = DateTime.UtcNow; Manifest.Metadata.TagIds.Add(TagId); StoreMetadata(Manifest); } if (Tag.Unique) { string ParentPath = VirtualFileSystem.GetParentPath(Manifest.VirtualPath); string NodeName = VirtualFileSystem.GetNodeName(Manifest.VirtualPath); List <string> Children = GetVirtualPathChildren(ParentPath); Tag DecayTag = TagRegistry.GetTagById(Tag.DecayTagId); foreach (string ChildName in Children) { if (ChildName == Manifest.VirtualPath) { continue; } BuildManifest ChildManifest = GetManifestByPath(ChildName); if (ChildManifest != null) { if (ChildManifest.Metadata != null && ChildManifest.Metadata.TagIds.Contains(TagId)) { UntagManifest(ChildManifest.Guid, TagId); if (DecayTag != null) { TagManifest(ChildManifest.Guid, DecayTag.Id); } } } } } }
/// <summary> /// /// </summary> private void StoreMetadata(BuildManifest Manifest) { string MetaFile = Path.Combine(RootPath, Manifest.Guid + ".manifest") + ".metadata"; Logger.Log(LogLevel.Info, LogCategory.Manifest, "Storing manifest metadata: {0}", MetaFile); if (Manifest.Metadata != null) { Manifest.Metadata.WriteToFile(MetaFile); } }
/// <summary> /// </summary> /// <param name="FilePath"></param> /// <returns></returns> public static BuildManifest FromByteArray(byte[] ByteArray) { BuildManifest Manifest = FileUtils.ReadFromArray <BuildManifest>(ByteArray); if (Manifest != null) { Manifest.UpgradeVersion(); Manifest.CacheBlockInfo(); Manifest.CacheSizeInfo(); } return(Manifest); }
/// <summary> /// </summary> public void PruneManifests() { if (Manifests.Count <= MaximumManifests) { return; } Logger.Log(LogLevel.Info, LogCategory.Manifest, "Pruning manifests to max limits"); // Create a flat list of folders and all builds they contain, we will prune starting from the // folders with the largest number of builds. Dictionary <string, List <BuildManifest> > Folders = new Dictionary <string, List <BuildManifest> >(); foreach (BuildManifest Manifest in Manifests) { string Folder = VirtualFileSystem.GetParentPath(Manifest.VirtualPath); if (!Folders.ContainsKey(Folder)) { Folders.Add(Folder, new List <BuildManifest>()); } List <BuildManifest> List = Folders[Folder]; List.Add(Manifest); } // Sort all folders. foreach (KeyValuePair <string, List <BuildManifest> > Entry in Folders) { Entry.Value.Sort((Item1, Item2) => - Item1.CreateTime.CompareTo(Item2.CreateTime)); } // Prune until we are back in space constraints. while (Manifests.Count > MaximumManifests) { // Find folder with the most entries. List <BuildManifest> Folder = null; foreach (KeyValuePair <string, List <BuildManifest> > Entry in Folders) { if (Folder == null || Entry.Value.Count > Folder.Count) { Folder = Entry.Value; } } // Last entry is oldests. BuildManifest Manifest = Folder[Folder.Count - 1]; Folder.RemoveAt(Folder.Count - 1); UnregisterManifest(Manifest.Guid); } }
/// <summary> /// </summary> /// <param name="FilePath"></param> /// <returns></returns> public static BuildManifest ReadFromFile(string FilePath) { BuildManifest Manifest = FileUtils.ReadFromBinaryFile <BuildManifest>(FilePath); if (Manifest != null) { Manifest.OriginalFilePath = FilePath; Manifest.UpgradeVersion(); Manifest.CacheSizeInfo(); Manifest.TrimBlockInfo(); } return(Manifest); }
/// <summary> /// </summary> public void RegisterManifest(BuildManifest Manifest) { string FilePath = Path.Combine(RootPath, Manifest.Guid + ".manifest"); Logger.Log(LogLevel.Info, LogCategory.Manifest, "Registering manifest: {0}", FilePath); if (Manifest.Metadata == null) { Manifest.Metadata = new BuildManifestMetadata(); } Manifest.WriteToFile(FilePath); AddManifest(Manifest); StoreMetadata(Manifest); }
/// <summary> /// /// </summary> /// <returns></returns> public List <BuildManifestFileDiff> Diff(BuildManifest Other) { LazyCacheBlockInfo(); if (Other != null) { Other.LazyCacheBlockInfo(); } List <BuildManifestFileDiff> Result = new List <BuildManifestFileDiff>(); if (Other == null) { foreach (BuildManifestFileInfo FileInfo in Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); FileDiff.Type = BuildManifestFileDiffType.Unchanged; FileDiff.FileInfo = FileInfo; Result.Add(FileDiff); } } else { // Get removed files. foreach (BuildManifestFileInfo FileInfo in Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); BuildManifestFileInfo OtherFileInfo = Other.GetFileInfo(FileInfo.Path); if (OtherFileInfo == null) { FileDiff.FileInfo = FileInfo; FileDiff.Type = BuildManifestFileDiffType.Removed; Result.Add(FileDiff); } } // Get added/modified files. foreach (BuildManifestFileInfo OtherFileInfo in Other.Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); FileDiff.FileInfo = OtherFileInfo; BuildManifestFileInfo FileInfo = GetFileInfo(OtherFileInfo.Path); if (FileInfo == null) { FileDiff.Type = BuildManifestFileDiffType.Added; } else { if (FileInfo.Checksum == OtherFileInfo.Checksum && FileInfo.Size == OtherFileInfo.Size) { FileDiff.Type = BuildManifestFileDiffType.Unchanged; } else { FileDiff.Type = BuildManifestFileDiffType.Modified; } } Result.Add(FileDiff); } } return(Result); }
/// <summary> /// </summary> public static BuildManifest BuildFromDirectory(Guid NewManifestId, string RootPath, string VirtualPath, AsyncIOQueue IOQueue, BuildManfiestProgressCallbackHandler Callback = null) { string[] FileNames = Directory.GetFiles(RootPath, "*", SearchOption.AllDirectories); long TotalSize = 0; // Try and order in the most efficient packing order. List <FileInfo> RemainingFiles = new List <FileInfo>(); List <FileInfo> OrderedList = new List <FileInfo>(); for (int i = 0; i < FileNames.Length; i++) { FileInfo Info = new FileInfo(FileNames[i]); RemainingFiles.Add(Info); TotalSize += Info.Length; } // Order main list from largest to smallest. RemainingFiles.Sort((Item1, Item2) => - Item1.Length.CompareTo(Item2.Length)); // Block size. long ActualBlockSize = DefaultBlockSize; // Try and add in optimal block packing format. long BlockIndex = 0; while (RemainingFiles.Count > 0) { FileInfo Info = RemainingFiles[0]; RemainingFiles.RemoveAt(0); OrderedList.Add(Info); //Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Name); long BlockCount = (Info.Length + (ActualBlockSize - 1)) / ActualBlockSize; long BytesRemaining = (Info.Length % ActualBlockSize) == 0 ? 0 : ActualBlockSize - Info.Length % ActualBlockSize; BlockIndex += BlockCount; long SubBlockCount = BlockCount; // Try and fit some smaller files into the remaining block space. for (int i = 0; i < RemainingFiles.Count && BytesRemaining > 0 && SubBlockCount < MaxSubBlockCount; i++) { FileInfo PotentialFile = RemainingFiles[i]; if (PotentialFile.Length <= BytesRemaining) { BytesRemaining -= PotentialFile.Length; SubBlockCount++; //Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, PotentialFile.Name); RemainingFiles.RemoveAt(i); OrderedList.Add(PotentialFile); i--; } } } // Our final list! FileInfo[] FileInfos = OrderedList.ToArray(); BuildManifest Manifest = new BuildManifest(); Manifest.Guid = NewManifestId; Manifest.VirtualPath = VirtualPath; Manifest.BlockCount = BlockIndex; Manifest.BlockChecksums = new uint[Manifest.BlockCount]; Manifest.CreateTime = DateTime.UtcNow; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums = new uint[Manifest.BlockCount]; #else Manifest.SparseBlockChecksums = null; #endif Manifest.Version = BuildManifest.CurrentVersion; Manifest.BlockSize = ActualBlockSize; List <Task> ChecksumTasks = new List <Task>(); int FileCounter = 0; int BlockCounter = 0; for (int i = 0; i < FileInfos.Length; i++) { Manifest.Files.Add(new BuildManifestFileInfo()); } for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { while (true) { int FileIndex = Interlocked.Increment(ref FileCounter) - 1; if (FileIndex >= FileInfos.Length) { break; } FileInfo SubFileInfo = FileInfos[FileIndex]; string SubFile = SubFileInfo.FullName; string RelativePath = SubFile.Substring(RootPath.Length).Trim('\\', '/'); if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback(RelativePath, Progress * 100); } } BuildManifestFileInfo ManifestFileInfo = new BuildManifestFileInfo(); ManifestFileInfo.Path = RelativePath; ManifestFileInfo.Size = new FileInfo(SubFile).Length; ManifestFileInfo.Checksum = FileUtils.GetChecksum(SubFile, null); lock (Manifest) { Manifest.Files[FileIndex] = ManifestFileInfo; } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } ChecksumTasks.Clear(); // Calculate which data goes in eahc block. Manifest.CacheBlockInfo(); Manifest.CacheSizeInfo(); Manifest.DebugCheck(); // Calculate checksum for each individual block. for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { byte[] Buffer = new byte[ActualBlockSize]; Stopwatch stopwatch = new Stopwatch(); while (true) { int CalculateBlockIndex = Interlocked.Increment(ref BlockCounter) - 1; if (CalculateBlockIndex >= Manifest.BlockCount) { break; } long BufferLength = 0; if (!Manifest.GetBlockData(CalculateBlockIndex, RootPath, IOQueue, Buffer, out BufferLength)) { // We should never end up in this situation when publishing ... Debug.Assert(false); } uint Checksum = 0; if (Manifest.Version >= 2) { Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength); } else { Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength); } Manifest.BlockChecksums[CalculateBlockIndex] = Checksum; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums[CalculateBlockIndex] = Crc32Slow.ComputeSparse(Buffer, (int)BufferLength); #endif if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback("Checksuming blocks", Progress * 100); } } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } return(Manifest); }
/// <summary> /// </summary> private void CacheBlockInfo() { // Reload content from disk cache. if (OriginalFilePath != "" && Files.Count == 0) { BuildManifest Manifest = FileUtils.ReadFromBinaryFile <BuildManifest>(OriginalFilePath); if (Manifest == null) { Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to reload trimmed block information from manifest, this may cause crashes if the data is unavailable: {0}", OriginalFilePath); return; } BlockChecksums = Manifest.BlockChecksums; SparseBlockChecksums = Manifest.SparseBlockChecksums; Files = Manifest.Files; } // Store block information. BlockInfo = new BuildManifestBlockInfo[BlockCount]; FilesByPath = new Dictionary <string, BuildManifestFileInfo>(); // Try and add in optimal block packing format. //Console.WriteLine("======================================== CACHING BLOCK INFO ================================"); long BlockIndex = 0; for (int fi = 0; fi < Files.Count;) { BuildManifestFileInfo Info = Files[fi]; Info.FirstBlockIndex = (int)BlockIndex; fi++; long BlockCount = (Info.Size + (BlockSize - 1)) / BlockSize; long BytesRemaining = (Info.Size % BlockSize) == 0 ? 0 : BlockSize - Info.Size % BlockSize; //Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Path); // Fill info for all the "full blocks" for this file. long Total = 0; for (int i = 0; i < BlockCount; i++) { BuildManifestSubBlockInfo SubBlock; SubBlock.File = Info; SubBlock.FileOffset = i * BlockSize; SubBlock.FileSize = Math.Min(BlockSize, Info.Size - SubBlock.FileOffset); SubBlock.OffsetInBlock = 0; Debug.Assert(SubBlock.FileOffset + SubBlock.FileSize <= Info.Size); Debug.Assert(BlockInfo[BlockIndex].SubBlocks == null); BlockInfo[BlockIndex].SubBlocks = new BuildManifestSubBlockInfo[1]; BlockInfo[BlockIndex].SubBlocks[0] = SubBlock; BlockInfo[BlockIndex].TotalSize += SubBlock.FileSize; Debug.Assert(BlockInfo[BlockIndex].TotalSize <= BlockSize); Total += SubBlock.FileSize; BlockIndex++; } Debug.Assert(Total == Info.Size); Info.LastBlockIndex = (int)BlockIndex - 1; int LastBlockIndex = Info.LastBlockIndex; // Fill remaining space with blocks. Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize); while (BytesRemaining > 0 && fi < Files.Count && BlockInfo[LastBlockIndex].SubBlocks.Length < BuildManifest.MaxSubBlockCount) { BuildManifestFileInfo NextInfo = Files[fi]; if (NextInfo.Size > BytesRemaining) { break; } if (NextInfo.Size > 0) { BuildManifestSubBlockInfo SubBlock; SubBlock.File = NextInfo; SubBlock.FileOffset = 0; SubBlock.FileSize = NextInfo.Size; SubBlock.OffsetInBlock = BlockInfo[LastBlockIndex].TotalSize; Array.Resize(ref BlockInfo[LastBlockIndex].SubBlocks, BlockInfo[LastBlockIndex].SubBlocks.Length + 1); BlockInfo[LastBlockIndex].SubBlocks[BlockInfo[LastBlockIndex].SubBlocks.Length - 1] = SubBlock; BlockInfo[LastBlockIndex].TotalSize += SubBlock.FileSize; Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize); //Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, SubBlock.File.Path); } NextInfo.FirstBlockIndex = (int)BlockIndex - 1; NextInfo.LastBlockIndex = (int)BlockIndex - 1; BytesRemaining -= NextInfo.Size; //Console.WriteLine("\tRemaining: {0}", BytesRemaining); fi++; } } }
/// <summary> /// </summary> /// <param name="Path"></param> public void Open(string Path, int InMaxManifests) { MaximumManifests = InMaxManifests; RootPath = Path; Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loading build manfiests from: {0}", Path); if (!Directory.Exists(RootPath)) { Directory.CreateDirectory(RootPath); } Stopwatch watch = new Stopwatch(); watch.Start(); string[] ManifestFilePaths = Directory.GetFiles(Path, "*.manifest", SearchOption.AllDirectories); List <Task> Tasks = new List <Task>(); List <BuildManifest> Results = new List <BuildManifest>(); foreach (string FilePath in ManifestFilePaths) { Tasks.Add(Task.Run(() => { try { Logger.Log(LogLevel.Verbose, LogCategory.Manifest, "Loading manifest: {0}", FilePath); BuildManifest Manifest = BuildManifest.ReadFromFile(FilePath); if (Manifest != null) { string MetaFile = FilePath + ".metadata"; if (File.Exists(MetaFile)) { Logger.Log(LogLevel.Verbose, LogCategory.Manifest, "Loading manifest metadata: {0}", MetaFile); Manifest.Metadata = BuildManifestMetadata.ReadFromFile(MetaFile); } if (Manifest.Metadata == null) { Manifest.Metadata = new BuildManifestMetadata(); } lock (Results) { Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loaded Manifest (Version {2}): {0} -> {1}", Manifest.Guid.ToString(), Manifest.VirtualPath, Manifest.Version); Results.Add(Manifest); } } } catch (Exception ex) { Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to read manifest '{0}', due to error {1}", FilePath, ex.Message); // File.Delete(FilePath); } })); } Task.WaitAll(Tasks.ToArray()); GC.Collect(); foreach (BuildManifest Manifest in Results) { AddManifest(Manifest); } watch.Stop(); Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loaded all manifests in {0}ms", watch.ElapsedMilliseconds); PruneManifests(); }