static void DecompressWorker(string RootPath, ConcurrentQueue <DependencyPack> DecompressQueue, Dictionary <DependencyPack, string> DownloadFileNames, Dictionary <string, List <DependencyBlob> > PackToBlobs, Dictionary <string, List <DependencyFile> > BlobToFiles, AsyncDownloadState State)
{
while (State.NumFilesRead < State.NumFiles)
{
// Remove the next file from the queue, or wait before polling again
DependencyPack NextPack;
if (!DecompressQueue.TryDequeue(out NextPack))
{
Thread.Sleep(100);
continue;
}
// Get the filename for the downloaded pack
string PackFileName = DownloadFileNames[NextPack];
// Extract all the files from this pack file to their final locations
foreach (DependencyBlob Blob in PackToBlobs[NextPack.Hash])
{
foreach (DependencyFile File in BlobToFiles[Blob.Hash])
{
string OutputFileName = Path.Combine(RootPath, File.Name);
try
{
ExtractBlob(PackFileName, Blob, OutputFileName);
}
catch (Exception Ex)
{
Interlocked.CompareExchange(ref State.LastDecompressError, String.Format("Failed to extract '{0}': {1}", OutputFileName, Ex.Message), null);
return;
}
Interlocked.Increment(ref State.NumFilesRead);
}
}
// Delete the pack file now that we're finished with it. Doesn't matter much if it fails.
try { System.IO.File.Delete(PackFileName); } catch (Exception) { }
}
}
static void DownloadWorker(string RootPath, ConcurrentQueue <DependencyPack> DownloadQueue, ConcurrentQueue <DependencyPack> DecompressQueue, Dictionary <DependencyPack, string> DownloadFileNames, Dictionary <string, List <DependencyBlob> > PackToBlobs, Dictionary <string, List <DependencyFile> > BlobToFiles, AsyncDownloadState State, int MaxRetries, string ProxyUrl)
{
int Retries = 0;
while (State.NumFilesRead < State.NumFiles)
{
// Remove the next file from the download queue, or wait before polling again
DependencyPack NextPack;
if (!DownloadQueue.TryDequeue(out NextPack))
{
Thread.Sleep(100);
continue;
}
// Get the temporary file to download to
string PackFileName = DownloadFileNames[NextPack];
// Format the URL for it
string Url = String.Format("http://cdn.unrealengine.com/dependencies/{0}/{1}", NextPack.RemotePath, NextPack.Hash);
// Try to download the file
long RollbackSize = 0;
try
{
// Download the file and queue it for decompression
DownloadFileAndVerifyHash(Url, ProxyUrl, PackFileName, NextPack.Hash, Size => { RollbackSize += Size; Interlocked.Add(ref State.NumBytesRead, Size); });
DecompressQueue.Enqueue(NextPack);
// If we were failing, decrement the number of failing threads
if (Retries > MaxRetries)
{
Interlocked.Decrement(ref State.NumFailingDownloads);
Retries = 0;
}
}
catch (Exception Ex)
{
// Rollback the byte count and add the file back into the download queue
Interlocked.Add(ref State.NumBytesRead, -RollbackSize);
DownloadQueue.Enqueue(NextPack);
// If we've retried enough times already, set the error message.
if (Retries++ == MaxRetries)
{
Interlocked.Increment(ref State.NumFailingDownloads);
State.LastDownloadError = String.Format("Failed to download '{0}' to '{1}': {2} ({3})", Url, PackFileName, Ex.Message, Ex.GetType().Name);
}
}
}
}
static bool DownloadDependencies(string RootPath, IEnumerable <DependencyFile> RequiredFiles, IEnumerable <DependencyBlob> Blobs, IEnumerable <DependencyPack> Packs, int NumThreads, int MaxRetries, string ProxyUrl)
{
// Make sure we can actually open the right number of connections
ServicePointManager.DefaultConnectionLimit = NumThreads;
// Build a lookup for the files that need updating from each blob
Dictionary <string, List <DependencyFile> > BlobToFiles = new Dictionary <string, List <DependencyFile> >();
foreach (DependencyFile RequiredFile in RequiredFiles)
{
List <DependencyFile> FileList;
if (!BlobToFiles.TryGetValue(RequiredFile.Hash, out FileList))
{
FileList = new List <DependencyFile>();
BlobToFiles.Add(RequiredFile.Hash, FileList);
}
FileList.Add(RequiredFile);
}
// Find all the required blobs
DependencyBlob[] RequiredBlobs = Blobs.Where(x => BlobToFiles.ContainsKey(x.Hash)).ToArray();
// Build a lookup for the files that need updating from each blob
Dictionary <string, List <DependencyBlob> > PackToBlobs = new Dictionary <string, List <DependencyBlob> >();
foreach (DependencyBlob RequiredBlob in RequiredBlobs)
{
List <DependencyBlob> BlobList = new List <DependencyBlob>();
if (!PackToBlobs.TryGetValue(RequiredBlob.PackHash, out BlobList))
{
BlobList = new List <DependencyBlob>();
PackToBlobs.Add(RequiredBlob.PackHash, BlobList);
}
BlobList.Add(RequiredBlob);
}
// Find all the required packs
DependencyPack[] RequiredPacks = Packs.Where(x => PackToBlobs.ContainsKey(x.Hash)).ToArray();
// Get temporary filenames for all the files we're going to download
Dictionary <DependencyPack, string> DownloadFileNames = new Dictionary <DependencyPack, string>();
foreach (DependencyPack Pack in RequiredPacks)
{
DownloadFileNames.Add(Pack, Path.GetTempFileName());
}
// Setup the async state
AsyncDownloadState State = new AsyncDownloadState();
State.NumFiles = RequiredFiles.Count();
long NumBytesTotal = RequiredPacks.Sum(x => x.CompressedSize);
ConcurrentQueue <DependencyPack> DownloadQueue = new ConcurrentQueue <DependencyPack>(RequiredPacks);
ConcurrentQueue <DependencyPack> DecompressQueue = new ConcurrentQueue <DependencyPack>();
// Create all the worker threads
Thread[] WorkerThreads = new Thread[NumThreads];
for (int Idx = 0; Idx < NumThreads; Idx++)
{
WorkerThreads[Idx] = new Thread(x => DownloadWorker(RootPath, DownloadQueue, DecompressQueue, DownloadFileNames, PackToBlobs, BlobToFiles, State, MaxRetries, ProxyUrl));
WorkerThreads[Idx].Start();
}
// Create the decompression thread
Thread DecompressionThread = new Thread(x => DecompressWorker(RootPath, DecompressQueue, DownloadFileNames, PackToBlobs, BlobToFiles, State));
DecompressionThread.Start();
// Tick the status message until we've finished or ended with an error. Use a circular buffer to average out the speed over time.
long[] NumBytesReadBuffer = new long[60];
for (int BufferIdx = 0, NumFilesReportedRead = 0; NumFilesReportedRead < State.NumFiles && State.NumFailingDownloads < NumThreads && State.LastDecompressError == null; BufferIdx = (BufferIdx + 1) % NumBytesReadBuffer.Length)
{
const int TickInterval = 100;
long NumBytesRead = Interlocked.Read(ref State.NumBytesRead);
float NumBytesPerSecond = (float)Math.Max(NumBytesRead - NumBytesReadBuffer[BufferIdx], 0) * 1000.0f / (NumBytesReadBuffer.Length * TickInterval);
NumFilesReportedRead = State.NumFilesRead;
Log.WriteStatus("Received {0}/{1} files ({2:0.0}/{3:0.0}mb; {4:0.00}mb/s; {5}%)...", NumFilesReportedRead, State.NumFiles, (NumBytesRead / (1024.0 * 1024.0)) + 0.0999999, (NumBytesTotal / (1024.0 * 1024.0)) + 0.0999999, (NumBytesPerSecond / (1024.0 * 1024.0)) + 0.0099, (NumBytesRead * 100) / NumBytesTotal);
NumBytesReadBuffer[BufferIdx] = NumBytesRead;
Thread.Sleep(TickInterval);
}
// If we finished with an error, try to clean up and return
if (State.NumFilesRead < State.NumFiles)
{
DecompressionThread.Abort();
foreach (Thread WorkerThread in WorkerThreads)
{
WorkerThread.Abort();
}
Log.WriteError("{0}", (State.LastDecompressError != null)? State.LastDecompressError : State.LastDownloadError);
foreach (string FileName in DownloadFileNames.Values)
{
try { File.Delete(FileName); } catch (Exception) { }
}
return(false);
}
// Join all the threads
DecompressionThread.Join();
foreach (Thread WorkerThread in WorkerThreads)
{
WorkerThread.Join();
}
Log.FlushStatus();
return(true);
}
