/// <summary> /// </summary> /// <param name="FilePath"></param> /// <returns></returns> public static BuildManifest FromByteArray(byte[] ByteArray) { BuildManifest Manifest = FileUtils.ReadFromArray <BuildManifest>(ByteArray); if (Manifest != null) { Manifest.UpgradeVersion(); Manifest.CacheBlockInfo(); Manifest.CacheSizeInfo(); } return(Manifest); }
/// <summary> /// </summary> public static BuildManifest BuildFromDirectory(Guid NewManifestId, string RootPath, string VirtualPath, AsyncIOQueue IOQueue, BuildManfiestProgressCallbackHandler Callback = null) { string[] FileNames = Directory.GetFiles(RootPath, "*", SearchOption.AllDirectories); long TotalSize = 0; // Try and order in the most efficient packing order. List <FileInfo> RemainingFiles = new List <FileInfo>(); List <FileInfo> OrderedList = new List <FileInfo>(); for (int i = 0; i < FileNames.Length; i++) { FileInfo Info = new FileInfo(FileNames[i]); RemainingFiles.Add(Info); TotalSize += Info.Length; } // Order main list from largest to smallest. RemainingFiles.Sort((Item1, Item2) => - Item1.Length.CompareTo(Item2.Length)); // Block size. long ActualBlockSize = DefaultBlockSize; // Try and add in optimal block packing format. long BlockIndex = 0; while (RemainingFiles.Count > 0) { FileInfo Info = RemainingFiles[0]; RemainingFiles.RemoveAt(0); OrderedList.Add(Info); //Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Name); long BlockCount = (Info.Length + (ActualBlockSize - 1)) / ActualBlockSize; long BytesRemaining = (Info.Length % ActualBlockSize) == 0 ? 0 : ActualBlockSize - Info.Length % ActualBlockSize; BlockIndex += BlockCount; long SubBlockCount = BlockCount; // Try and fit some smaller files into the remaining block space. for (int i = 0; i < RemainingFiles.Count && BytesRemaining > 0 && SubBlockCount < MaxSubBlockCount; i++) { FileInfo PotentialFile = RemainingFiles[i]; if (PotentialFile.Length <= BytesRemaining) { BytesRemaining -= PotentialFile.Length; SubBlockCount++; //Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, PotentialFile.Name); RemainingFiles.RemoveAt(i); OrderedList.Add(PotentialFile); i--; } } } // Our final list! FileInfo[] FileInfos = OrderedList.ToArray(); BuildManifest Manifest = new BuildManifest(); Manifest.Guid = NewManifestId; Manifest.VirtualPath = VirtualPath; Manifest.BlockCount = BlockIndex; Manifest.BlockChecksums = new uint[Manifest.BlockCount]; Manifest.CreateTime = DateTime.UtcNow; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums = new uint[Manifest.BlockCount]; #else Manifest.SparseBlockChecksums = null; #endif Manifest.Version = BuildManifest.CurrentVersion; Manifest.BlockSize = ActualBlockSize; List <Task> ChecksumTasks = new List <Task>(); int FileCounter = 0; int BlockCounter = 0; for (int i = 0; i < FileInfos.Length; i++) { Manifest.Files.Add(new BuildManifestFileInfo()); } for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { while (true) { int FileIndex = Interlocked.Increment(ref FileCounter) - 1; if (FileIndex >= FileInfos.Length) { break; } FileInfo SubFileInfo = FileInfos[FileIndex]; string SubFile = SubFileInfo.FullName; string RelativePath = SubFile.Substring(RootPath.Length).Trim('\\', '/'); if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback(RelativePath, Progress * 100); } } BuildManifestFileInfo ManifestFileInfo = new BuildManifestFileInfo(); ManifestFileInfo.Path = RelativePath; ManifestFileInfo.Size = new FileInfo(SubFile).Length; ManifestFileInfo.Checksum = FileUtils.GetChecksum(SubFile, null); lock (Manifest) { Manifest.Files[FileIndex] = ManifestFileInfo; } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } ChecksumTasks.Clear(); // Calculate which data goes in eahc block. Manifest.CacheBlockInfo(); Manifest.CacheSizeInfo(); Manifest.DebugCheck(); // Calculate checksum for each individual block. for (int i = 0; i < Environment.ProcessorCount; i++) { ChecksumTasks.Add( Task.Run( () => { byte[] Buffer = new byte[ActualBlockSize]; Stopwatch stopwatch = new Stopwatch(); while (true) { int CalculateBlockIndex = Interlocked.Increment(ref BlockCounter) - 1; if (CalculateBlockIndex >= Manifest.BlockCount) { break; } long BufferLength = 0; if (!Manifest.GetBlockData(CalculateBlockIndex, RootPath, IOQueue, Buffer, out BufferLength)) { // We should never end up in this situation when publishing ... Debug.Assert(false); } uint Checksum = 0; if (Manifest.Version >= 2) { Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength); } else { Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength); } Manifest.BlockChecksums[CalculateBlockIndex] = Checksum; #if USE_SPARSE_CHECKSUMS Manifest.SparseBlockChecksums[CalculateBlockIndex] = Crc32Slow.ComputeSparse(Buffer, (int)BufferLength); #endif if (Callback != null) { lock (Callback) { float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount); Callback("Checksuming blocks", Progress * 100); } } } } ) ); } foreach (Task task in ChecksumTasks) { task.Wait(); } return(Manifest); }