/// <summary> /// </summary> /// <param name="Path"></param> /// <returns></returns> public List <int> GetFileBlocks(string Path) { LazyCacheBlockInfo(); List <int> Result = new List <int>(); BuildManifestFileInfo FileInfo = GetFileInfo(Path); if (FileInfo == null) { return(Result); } for (int i = FileInfo.FirstBlockIndex; i <= FileInfo.LastBlockIndex; i++) { Result.Add(i); } return(Result); }
/// <summary> /// </summary> public List <int> Validate(string RootPath, RateTracker Tracker, AsyncIOQueue IOQueue, BuildManfiestValidateProgressCallbackHandler Callback = null) { LazyCacheBlockInfo(); List <int> FailedBlocks = new List <int>(); try { LockBlockInfo(); int TaskCount = Environment.ProcessorCount; Task[] FileTasks = new Task[TaskCount]; int BlockCounter = 0; long BytesValidated = 0; bool Aborted = false; // Check the size of each file. for (int i = 0; i < Files.Count; i++) { BuildManifestFileInfo FileInfo = Files[i]; string FilePath = Path.Combine(RootPath, FileInfo.Path); string DirPath = Path.GetDirectoryName(FilePath); if (!Directory.Exists(DirPath)) { Directory.CreateDirectory(DirPath); Logger.Log(LogLevel.Warning, LogCategory.Manifest, "Expected directory {0} does not exist, creating.", DirPath); } FileInfo Info = new FileInfo(FilePath); if (!Info.Exists || Info.Length != FileInfo.Size) { using (FileStream Stream = new FileStream(FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) { Logger.Log(LogLevel.Warning, LogCategory.Manifest, "File {0} is not of expected length {1} (is {2}) settting length.", FilePath, FileInfo.Size, Info.Length); Stream.SetLength(FileInfo.Size); } } } // Check each individual block of data for validity. for (int i = 0; i < TaskCount; i++) { FileTasks[i] = Task.Run( () => { byte[] Buffer = new byte[BlockSize]; while (!Aborted) { int BlockIndex = Interlocked.Increment(ref BlockCounter) - 1; if (BlockIndex >= BlockCount) { break; } BuildManifestBlockInfo BInfo = BlockInfo[BlockIndex]; long BufferLength = 0; bool Success = GetBlockData(BlockIndex, RootPath, IOQueue, Buffer, out BufferLength); uint Checksum = 0; if (Version >= 2) { Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength); } else { Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength); } if (!Success || BlockChecksums[BlockIndex] != Checksum) { Logger.Log(LogLevel.Warning, LogCategory.Manifest, "Block {0} failed checksum, block contains following sub-blocks:", BlockIndex); for (int SubBlock = 0; SubBlock < BInfo.SubBlocks.Length; SubBlock++) { BuildManifestSubBlockInfo SubBInfo = BInfo.SubBlocks[SubBlock]; Logger.Log(LogLevel.Warning, LogCategory.Manifest, "\tfile={0} offset={1} size={2}", SubBInfo.File.Path, SubBInfo.FileOffset, SubBInfo.FileSize); } lock (FailedBlocks) { FailedBlocks.Add(BlockIndex); } } Interlocked.Add(ref BytesValidated, BInfo.TotalSize); if (Callback != null) { if (!Callback.Invoke(BytesValidated, TotalSize, Guid, BlockIndex)) { Aborted = true; } } } } ); } Task.WaitAll(FileTasks); } finally { UnlockBlockInfo(); } return(FailedBlocks); }
/// <summary> /// </summary> public static BuildManifest BuildFromDirectory(Guid NewManifestId, string RootPath, string VirtualPath, AsyncIOQueue IOQueue, BuildManfiestProgressCallbackHandler Callback = null) { string[] FileNames = Directory.GetFiles(RootPath, "*", SearchOption.AllDirectories); long TotalSize = 0; // Try and order in the most efficient packing order. List <FileInfo> RemainingFiles = new List <FileInfo>(); List <FileInfo> OrderedList = new List <FileInfo>(); for (int i = 0; i < FileNames.Length; i++) { FileInfo Info = new FileInfo(FileNames[i]); RemainingFiles.Add(Info); TotalSize += Info.Length; } // Order main list from largest to smallest. RemainingFiles.Sort((Item1, Item2) => - Item1.Length.CompareTo(Item2.Length)); // Block size. long ActualBlockSize = DefaultBlockSize; // Try and add in optimal block packing format. long BlockIndex = 0; while (RemainingFiles.Count > 0) { FileInfo Info = RemainingFiles[0]; RemainingFiles.RemoveAt(0); OrderedList.Add(Info); //Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Name); long BlockCount = (Info.Length + (ActualBlockSize - 1)) / ActualBlockSize; long BytesRemaining = (Info.Length % ActualBlockSize) == 0 ? 0 : ActualBlockSize - Info.Length % ActualBlockSize; BlockIndex += BlockCount; long SubBlockCount = BlockCount; // Try and fit some smaller files into the remaining block space. for (int i = 0; i < RemainingFiles.Count && BytesRemaining > 0 && SubBlockCount < MaxSubBlockCount; i++) { FileInfo PotentialFile = RemainingFiles[i]; if (PotentialFile.Length <= BytesRemaining) { BytesRemaining -= PotentialFile.Length; SubBlockCount++; //Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, PotentialFile.Name); RemainingFiles.RemoveAt(i); OrderedList.Add(PotentialFile); i--; } } } // Our final list! FileInfo[] FileInfos = OrderedList.ToArray(); BuildManifest Manifest = new BuildManifest(); Manifest.Guid = NewManifestId; Manifest.VirtualPath = VirtualPath; Manifest.BlockCount = BlockIndex; Manifest.BlockChecksums = new uint[Manifest.BlockCount]; Manifest.CreateTime = DateTime.UtcNow; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums = new uint[Manifest.BlockCount]; #else Manifest.SparseBlockChecksums = null; #endif Manifest.Version = BuildManifest.CurrentVersion; Manifest.BlockSize = ActualBlockSize; List <Task> ChecksumTasks = new List <Task>(); int FileCounter = 0; int BlockCounter = 0; for (int i = 0; i < FileInfos.Length; i++) { Manifest.Files.Add(new BuildManifestFileInfo()); } for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { while (true) { int FileIndex = Interlocked.Increment(ref FileCounter) - 1; if (FileIndex >= FileInfos.Length) { break; } FileInfo SubFileInfo = FileInfos[FileIndex]; string SubFile = SubFileInfo.FullName; string RelativePath = SubFile.Substring(RootPath.Length).Trim('\\', '/'); if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback(RelativePath, Progress * 100); } } BuildManifestFileInfo ManifestFileInfo = new BuildManifestFileInfo(); ManifestFileInfo.Path = RelativePath; ManifestFileInfo.Size = new FileInfo(SubFile).Length; ManifestFileInfo.Checksum = FileUtils.GetChecksum(SubFile, null); lock (Manifest) { Manifest.Files[FileIndex] = ManifestFileInfo; } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } ChecksumTasks.Clear(); // Calculate which data goes in eahc block. Manifest.CacheBlockInfo(); Manifest.CacheSizeInfo(); Manifest.DebugCheck(); // Calculate checksum for each individual block. for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { byte[] Buffer = new byte[ActualBlockSize]; Stopwatch stopwatch = new Stopwatch(); while (true) { int CalculateBlockIndex = Interlocked.Increment(ref BlockCounter) - 1; if (CalculateBlockIndex >= Manifest.BlockCount) { break; } long BufferLength = 0; if (!Manifest.GetBlockData(CalculateBlockIndex, RootPath, IOQueue, Buffer, out BufferLength)) { // We should never end up in this situation when publishing ... Debug.Assert(false); } uint Checksum = 0; if (Manifest.Version >= 2) { Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength); } else { Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength); } Manifest.BlockChecksums[CalculateBlockIndex] = Checksum; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums[CalculateBlockIndex] = Crc32Slow.ComputeSparse(Buffer, (int)BufferLength); #endif if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback("Checksuming blocks", Progress * 100); } } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } return(Manifest); }
/// <summary> /// /// </summary> /// <returns></returns> public List <BuildManifestFileDiff> Diff(BuildManifest Other) { LazyCacheBlockInfo(); if (Other != null) { Other.LazyCacheBlockInfo(); } List <BuildManifestFileDiff> Result = new List <BuildManifestFileDiff>(); if (Other == null) { foreach (BuildManifestFileInfo FileInfo in Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); FileDiff.Type = BuildManifestFileDiffType.Unchanged; FileDiff.FileInfo = FileInfo; Result.Add(FileDiff); } } else { // Get removed files. foreach (BuildManifestFileInfo FileInfo in Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); BuildManifestFileInfo OtherFileInfo = Other.GetFileInfo(FileInfo.Path); if (OtherFileInfo == null) { FileDiff.FileInfo = FileInfo; FileDiff.Type = BuildManifestFileDiffType.Removed; Result.Add(FileDiff); } } // Get added/modified files. foreach (BuildManifestFileInfo OtherFileInfo in Other.Files) { BuildManifestFileDiff FileDiff = new BuildManifestFileDiff(); FileDiff.FileInfo = OtherFileInfo; BuildManifestFileInfo FileInfo = GetFileInfo(OtherFileInfo.Path); if (FileInfo == null) { FileDiff.Type = BuildManifestFileDiffType.Added; } else { if (FileInfo.Checksum == OtherFileInfo.Checksum && FileInfo.Size == OtherFileInfo.Size) { FileDiff.Type = BuildManifestFileDiffType.Unchanged; } else { FileDiff.Type = BuildManifestFileDiffType.Modified; } } Result.Add(FileDiff); } } return(Result); }
/// <summary> /// </summary> private void CacheBlockInfo() { // Reload content from disk cache. if (OriginalFilePath != "" && Files.Count == 0) { BuildManifest Manifest = FileUtils.ReadFromBinaryFile <BuildManifest>(OriginalFilePath); if (Manifest == null) { Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to reload trimmed block information from manifest, this may cause crashes if the data is unavailable: {0}", OriginalFilePath); return; } BlockChecksums = Manifest.BlockChecksums; SparseBlockChecksums = Manifest.SparseBlockChecksums; Files = Manifest.Files; } // Store block information. BlockInfo = new BuildManifestBlockInfo[BlockCount]; FilesByPath = new Dictionary <string, BuildManifestFileInfo>(); // Try and add in optimal block packing format. //Console.WriteLine("======================================== CACHING BLOCK INFO ================================"); long BlockIndex = 0; for (int fi = 0; fi < Files.Count;) { BuildManifestFileInfo Info = Files[fi]; Info.FirstBlockIndex = (int)BlockIndex; fi++; long BlockCount = (Info.Size + (BlockSize - 1)) / BlockSize; long BytesRemaining = (Info.Size % BlockSize) == 0 ? 0 : BlockSize - Info.Size % BlockSize; //Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Path); // Fill info for all the "full blocks" for this file. long Total = 0; for (int i = 0; i < BlockCount; i++) { BuildManifestSubBlockInfo SubBlock; SubBlock.File = Info; SubBlock.FileOffset = i * BlockSize; SubBlock.FileSize = Math.Min(BlockSize, Info.Size - SubBlock.FileOffset); SubBlock.OffsetInBlock = 0; Debug.Assert(SubBlock.FileOffset + SubBlock.FileSize <= Info.Size); Debug.Assert(BlockInfo[BlockIndex].SubBlocks == null); BlockInfo[BlockIndex].SubBlocks = new BuildManifestSubBlockInfo[1]; BlockInfo[BlockIndex].SubBlocks[0] = SubBlock; BlockInfo[BlockIndex].TotalSize += SubBlock.FileSize; Debug.Assert(BlockInfo[BlockIndex].TotalSize <= BlockSize); Total += SubBlock.FileSize; BlockIndex++; } Debug.Assert(Total == Info.Size); Info.LastBlockIndex = (int)BlockIndex - 1; int LastBlockIndex = Info.LastBlockIndex; // Fill remaining space with blocks. Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize); while (BytesRemaining > 0 && fi < Files.Count && BlockInfo[LastBlockIndex].SubBlocks.Length < BuildManifest.MaxSubBlockCount) { BuildManifestFileInfo NextInfo = Files[fi]; if (NextInfo.Size > BytesRemaining) { break; } if (NextInfo.Size > 0) { BuildManifestSubBlockInfo SubBlock; SubBlock.File = NextInfo; SubBlock.FileOffset = 0; SubBlock.FileSize = NextInfo.Size; SubBlock.OffsetInBlock = BlockInfo[LastBlockIndex].TotalSize; Array.Resize(ref BlockInfo[LastBlockIndex].SubBlocks, BlockInfo[LastBlockIndex].SubBlocks.Length + 1); BlockInfo[LastBlockIndex].SubBlocks[BlockInfo[LastBlockIndex].SubBlocks.Length - 1] = SubBlock; BlockInfo[LastBlockIndex].TotalSize += SubBlock.FileSize; Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize); //Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, SubBlock.File.Path); } NextInfo.FirstBlockIndex = (int)BlockIndex - 1; NextInfo.LastBlockIndex = (int)BlockIndex - 1; BytesRemaining -= NextInfo.Size; //Console.WriteLine("\tRemaining: {0}", BytesRemaining); fi++; } } }