/// <summary> /// /// </summary> /// <param name="URL">The URL of the resource to download.</param> /// <param name="Parts">Number of parts to download file as</param> /// <param name="outFile">Outfile name, will be auto generated if /// null.</param> <param name="OnUpdate">Progress OnComplete function</param> /// <returns></returns> public async Task DownloadFile(string URL, double Parts, string outFile = null, Action <int> OnUpdate = null) { #region Variables EventfulConcurrentQueue <FileChunk> asyncTasks; TransformManyBlock <IEnumerable <FileChunk>, FileChunk> getFileChunk; TransformBlock <FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> > getStream; ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> > writeStream; // Get response length responseLength = (await WebRequest.Create(URL).GetResponseAsync()).ContentLength; // Calculate Part size var partSize = (long)Math.Round(responseLength / Parts); // Get the content ranges to download var pieces = GetChunkList(partSize, responseLength); // Stores the default console title for later restore var defaultTitle = Console.Title; // URL To uri var uri = new Uri(URL); // Outfile name for later null check var filename = ""; if (outFile == null) { filename = Path.GetFileName(uri.LocalPath); } else { filename = outFile; } #endregion Console.WriteLine(responseLength.ToString(CultureInfo.InvariantCulture) + " TOTAL SIZE"); Console.WriteLine(partSize.ToString(CultureInfo.InvariantCulture) + " PART SIZE" + "\n"); // Set max threads to those supported by system SetMaxThreads(); try { using (var progress = new ProgressBar()) { // Using custom concurrent queue to implement Enqueue and Dequeue Events asyncTasks = GetTaskList(progress, Parts, OnUpdate); Console.Title = "CHUNKS DONE"; // Transform many to get from List<Filechunk> => Filechunk essentially // iterating getFileChunk = new TransformManyBlock <IEnumerable <FileChunk>, FileChunk>( chunk => chunk, new ExecutionDataflowBlockOptions()); // Gets the request stream from the filechunk getStream = new TransformBlock <FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> >( piece => { var newTask = getStreamTask(piece, responseLength, uri, asyncTasks); return(newTask); }, new ExecutionDataflowBlockOptions { BoundedCapacity = Environment.ProcessorCount, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount, // Parallelize on all cores }); // Writes the request stream to a tempfile writeStream = new ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> >( async task => { using ( var streamToRead = await task.Item1.Result.Content.ReadAsStreamAsync()) { using (var fileToWriteTo = File.Open( task.Item2.TempFileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite)) { fileToWriteTo.Position = 0; await streamToRead.CopyToAsync(fileToWriteTo, (int)partSize, CancellationToken.None); } var s = new FileChunk(); Interlocked.Add(ref TasksDone, 1); asyncTasks.TryDequeue(out s); } GC.Collect(0, GCCollectionMode.Forced); }, new ExecutionDataflowBlockOptions { BoundedCapacity = Environment.ProcessorCount, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount, // Parallelize on all cores }); // Propage errors and completion var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; // Build the data flow pipeline getFileChunk.LinkTo(getStream, linkOptions); getStream.LinkTo(writeStream, linkOptions); // Post the file pieces getFileChunk.Post(pieces); getFileChunk.Complete(); // Write all the streams await writeStream.Completion.ContinueWith( task => { // If all the tasks are done, Join the temp files if (asyncTasks.Count == 0) { CombineMultipleFilesIntoSingleFile(pieces, filename); } }, CancellationToken.None, TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Current); } // Restore the original title Console.Title = defaultTitle; } catch (Exception ex) { Console.WriteLine(ex.Message); // Delete the tempfiles if there's an error foreach (var piece in pieces) { try { File.Delete(piece.TempFileName); } catch (FileNotFoundException) { } } } }
public async Task DownloadByteArray(string URL, double parts, Action <byte[]> OnComplete, Action <int> OnUpdate = null) { var responseLength = (await WebRequest.Create(URL).GetResponseAsync()).ContentLength; var partSize = (long)Math.Floor(responseLength / parts); var pieces = new List <FileChunk>(); ThreadPool.GetMaxThreads(out int maxworkerThreads, out int maxconcurrentActiveRequests); bool changeSucceeded = ThreadPool.SetMaxThreads(maxworkerThreads, maxconcurrentActiveRequests); Console.WriteLine(responseLength.ToString(CultureInfo.InvariantCulture) + " TOTAL SIZE"); Console.WriteLine(partSize.ToString(CultureInfo.InvariantCulture) + " PART SIZE" + "\n"); try { // Using our custom progressbar using (var progress = new ProgressBar()) { using (var ms = new MemoryStream()) { ms.SetLength(responseLength); // Using custom concurrent queue to implement Enqueue and Dequeue // Events var asyncTasks = new EventfulConcurrentQueue <FileChunk>(); // Delegate for Dequeue asyncTasks.ItemDequeued += delegate { // Tasks done holds the count of the tasks done // Parts *2 because there are Parts number of Enqueue AND Dequeue // operations if (OnUpdate == null) { progress.Report(TasksDone / (parts * 2)); Thread.Sleep(20); } else { OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; // Delegate for Enqueue asyncTasks.ItemEnqueued += delegate { if (OnUpdate == null) { progress.Report(TasksDone / (parts * 2)); Thread.Sleep(20); } else { OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; // GetResponseAsync deadlocks for some reason so switched to // HttpClient instead var client = new HttpClient( // Use our custom Retry handler, with a max retry value of 10 new RetryHandler(new HttpClientHandler(), 10)) { MaxResponseContentBufferSize = 1000000000 }; client.DefaultRequestHeaders.ConnectionClose = false; client.Timeout = Timeout.InfiniteTimeSpan; // Variable to hold the old loop end var previous = 0; // Loop to add all the events to the queue for (var i = (int)partSize; i <= responseLength; i += (int)partSize) { Console.Title = "WRITING CHUNKS..."; if (i + partSize < responseLength) { // Start and end values for the chunk var start = previous; var currentEnd = i; pieces.Add(new FileChunk(start, currentEnd)); // Set the start of the next loop to be the current end previous = currentEnd; } else { // Start and end values for the chunk var start = previous; var currentEnd = i; pieces.Add(new FileChunk(start, (int)responseLength)); // Set the start of the next loop to be the current end previous = currentEnd; } } var getFileChunk = new TransformManyBlock <IEnumerable <FileChunk>, FileChunk>( chunk => chunk, new ExecutionDataflowBlockOptions() { BoundedCapacity = Int32.MaxValue, // Cap the item count MaxDegreeOfParallelism = Environment .ProcessorCount, // Parallelize on all cores }); var getStream = new TransformBlock < FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> >( piece => { Console.Title = "STREAMING...."; // Open a http request with the range var request = new HttpRequestMessage { RequestUri = new Uri(URL) }; request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End); // Send the request var downloadTask = client.SendAsync( request, HttpCompletionOption.ResponseContentRead); // Use interlocked to increment Tasks done by one Interlocked.Add(ref TasksDone, 1); asyncTasks.Enqueue(piece); return(new Tuple <Task <HttpResponseMessage>, FileChunk>( downloadTask, piece)); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount, // Parallelize on all cores }); var writeStream = new ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> >( async tuple => { var buffer = new byte[tuple.Item2.End - tuple.Item2.Start]; using (var stream = await tuple.Item1.Result.Content .ReadAsStreamAsync()) { await stream.ReadAsync(buffer, 0, buffer.Length); } lock (ms) { ms.Position = tuple.Item2.Start; ms.Write(buffer, 0, buffer.Length); } var s = new FileChunk(); asyncTasks.TryDequeue(out s); Interlocked.Add(ref TasksDone, 1); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment .ProcessorCount, // Parallelize on all cores }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; getFileChunk.LinkTo(getStream, linkOptions); getStream.LinkTo(writeStream, linkOptions); getFileChunk.Post(pieces); getFileChunk.Complete(); await writeStream.Completion.ContinueWith(task => { if (asyncTasks.Count == 0) { ms.Flush(); ms.Close(); OnComplete?.Invoke(ms.ToArray()); } }); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } }