static void DecompressWorker(string RootPath, ConcurrentQueue <DependencyPack> DecompressQueue, Dictionary <DependencyPack, string> DownloadFileNames, Dictionary <string, List <DependencyBlob> > PackToBlobs, Dictionary <string, List <DependencyFile> > BlobToFiles, AsyncDownloadState State) { while (State.NumFilesRead < State.NumFiles) { // Remove the next file from the queue, or wait before polling again DependencyPack NextPack; if (!DecompressQueue.TryDequeue(out NextPack)) { Thread.Sleep(100); continue; } // Get the filename for the downloaded pack string PackFileName = DownloadFileNames[NextPack]; // Extract all the files from this pack file to their final locations foreach (DependencyBlob Blob in PackToBlobs[NextPack.Hash]) { foreach (DependencyFile File in BlobToFiles[Blob.Hash]) { string OutputFileName = Path.Combine(RootPath, File.Name); try { ExtractBlob(PackFileName, Blob, OutputFileName); } catch (Exception Ex) { Interlocked.CompareExchange(ref State.LastDecompressError, String.Format("Failed to extract '{0}': {1}", OutputFileName, Ex.Message), null); return; } Interlocked.Increment(ref State.NumFilesRead); } } // Delete the pack file now that we're finished with it. Doesn't matter much if it fails. try { System.IO.File.Delete(PackFileName); } catch (Exception) { } } }
static void DownloadWorker(string RootPath, ConcurrentQueue <DependencyPack> DownloadQueue, ConcurrentQueue <DependencyPack> DecompressQueue, Dictionary <DependencyPack, string> DownloadFileNames, Dictionary <string, List <DependencyBlob> > PackToBlobs, Dictionary <string, List <DependencyFile> > BlobToFiles, AsyncDownloadState State, int MaxRetries, string ProxyUrl) { int Retries = 0; while (State.NumFilesRead < State.NumFiles) { // Remove the next file from the download queue, or wait before polling again DependencyPack NextPack; if (!DownloadQueue.TryDequeue(out NextPack)) { Thread.Sleep(100); continue; } // Get the temporary file to download to string PackFileName = DownloadFileNames[NextPack]; // Format the URL for it string Url = String.Format("http://cdn.unrealengine.com/dependencies/{0}/{1}", NextPack.RemotePath, NextPack.Hash); // Try to download the file long RollbackSize = 0; try { // Download the file and queue it for decompression DownloadFileAndVerifyHash(Url, ProxyUrl, PackFileName, NextPack.Hash, Size => { RollbackSize += Size; Interlocked.Add(ref State.NumBytesRead, Size); }); DecompressQueue.Enqueue(NextPack); // If we were failing, decrement the number of failing threads if (Retries > MaxRetries) { Interlocked.Decrement(ref State.NumFailingDownloads); Retries = 0; } } catch (Exception Ex) { // Rollback the byte count and add the file back into the download queue Interlocked.Add(ref State.NumBytesRead, -RollbackSize); DownloadQueue.Enqueue(NextPack); // If we've retried enough times already, set the error message. if (Retries++ == MaxRetries) { Interlocked.Increment(ref State.NumFailingDownloads); State.LastDownloadError = String.Format("Failed to download '{0}' to '{1}': {2} ({3})", Url, PackFileName, Ex.Message, Ex.GetType().Name); } } } }
static bool DownloadDependencies(string RootPath, IEnumerable <DependencyFile> RequiredFiles, IEnumerable <DependencyBlob> Blobs, IEnumerable <DependencyPack> Packs, int NumThreads, int MaxRetries, string ProxyUrl) { // Make sure we can actually open the right number of connections ServicePointManager.DefaultConnectionLimit = NumThreads; // Build a lookup for the files that need updating from each blob Dictionary <string, List <DependencyFile> > BlobToFiles = new Dictionary <string, List <DependencyFile> >(); foreach (DependencyFile RequiredFile in RequiredFiles) { List <DependencyFile> FileList; if (!BlobToFiles.TryGetValue(RequiredFile.Hash, out FileList)) { FileList = new List <DependencyFile>(); BlobToFiles.Add(RequiredFile.Hash, FileList); } FileList.Add(RequiredFile); } // Find all the required blobs DependencyBlob[] RequiredBlobs = Blobs.Where(x => BlobToFiles.ContainsKey(x.Hash)).ToArray(); // Build a lookup for the files that need updating from each blob Dictionary <string, List <DependencyBlob> > PackToBlobs = new Dictionary <string, List <DependencyBlob> >(); foreach (DependencyBlob RequiredBlob in RequiredBlobs) { List <DependencyBlob> BlobList = new List <DependencyBlob>(); if (!PackToBlobs.TryGetValue(RequiredBlob.PackHash, out BlobList)) { BlobList = new List <DependencyBlob>(); PackToBlobs.Add(RequiredBlob.PackHash, BlobList); } BlobList.Add(RequiredBlob); } // Find all the required packs DependencyPack[] RequiredPacks = Packs.Where(x => PackToBlobs.ContainsKey(x.Hash)).ToArray(); // Get temporary filenames for all the files we're going to download Dictionary <DependencyPack, string> DownloadFileNames = new Dictionary <DependencyPack, string>(); foreach (DependencyPack Pack in RequiredPacks) { DownloadFileNames.Add(Pack, Path.GetTempFileName()); } // Setup the async state AsyncDownloadState State = new AsyncDownloadState(); State.NumFiles = RequiredFiles.Count(); long NumBytesTotal = RequiredPacks.Sum(x => x.CompressedSize); ConcurrentQueue <DependencyPack> DownloadQueue = new ConcurrentQueue <DependencyPack>(RequiredPacks); ConcurrentQueue <DependencyPack> DecompressQueue = new ConcurrentQueue <DependencyPack>(); // Create all the worker threads Thread[] WorkerThreads = new Thread[NumThreads]; for (int Idx = 0; Idx < NumThreads; Idx++) { WorkerThreads[Idx] = new Thread(x => DownloadWorker(RootPath, DownloadQueue, DecompressQueue, DownloadFileNames, PackToBlobs, BlobToFiles, State, MaxRetries, ProxyUrl)); WorkerThreads[Idx].Start(); } // Create the decompression thread Thread DecompressionThread = new Thread(x => DecompressWorker(RootPath, DecompressQueue, DownloadFileNames, PackToBlobs, BlobToFiles, State)); DecompressionThread.Start(); // Tick the status message until we've finished or ended with an error. Use a circular buffer to average out the speed over time. long[] NumBytesReadBuffer = new long[60]; for (int BufferIdx = 0, NumFilesReportedRead = 0; NumFilesReportedRead < State.NumFiles && State.NumFailingDownloads < NumThreads && State.LastDecompressError == null; BufferIdx = (BufferIdx + 1) % NumBytesReadBuffer.Length) { const int TickInterval = 100; long NumBytesRead = Interlocked.Read(ref State.NumBytesRead); float NumBytesPerSecond = (float)Math.Max(NumBytesRead - NumBytesReadBuffer[BufferIdx], 0) * 1000.0f / (NumBytesReadBuffer.Length * TickInterval); NumFilesReportedRead = State.NumFilesRead; Log.WriteStatus("Received {0}/{1} files ({2:0.0}/{3:0.0}mb; {4:0.00}mb/s; {5}%)...", NumFilesReportedRead, State.NumFiles, (NumBytesRead / (1024.0 * 1024.0)) + 0.0999999, (NumBytesTotal / (1024.0 * 1024.0)) + 0.0999999, (NumBytesPerSecond / (1024.0 * 1024.0)) + 0.0099, (NumBytesRead * 100) / NumBytesTotal); NumBytesReadBuffer[BufferIdx] = NumBytesRead; Thread.Sleep(TickInterval); } // If we finished with an error, try to clean up and return if (State.NumFilesRead < State.NumFiles) { DecompressionThread.Abort(); foreach (Thread WorkerThread in WorkerThreads) { WorkerThread.Abort(); } Log.WriteError("{0}", (State.LastDecompressError != null)? State.LastDecompressError : State.LastDownloadError); foreach (string FileName in DownloadFileNames.Values) { try { File.Delete(FileName); } catch (Exception) { } } return(false); } // Join all the threads DecompressionThread.Join(); foreach (Thread WorkerThread in WorkerThreads) { WorkerThread.Join(); } Log.FlushStatus(); return(true); }