GetTaskList(ProgressBar progress, double parts, Action <int> progressCallback = null) { var asyncTasks = new EventfulConcurrentQueue <FileChunk>(); // Delegate for Dequeue asyncTasks.ItemDequeued += delegate { // Tasks done holds the count of the tasks done // Parts *2 because there are Parts number of Enqueue AND Dequeue // operations if (progressCallback == null) { progress.Report(TasksDone / (parts * 2)); } else { progressCallback?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; // Delegate for Enqueue asyncTasks.ItemEnqueued += delegate { if (progressCallback == null) { progress.Report(TasksDone / (parts * 2)); } else { progressCallback?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; return(asyncTasks); }
getStreamTask(FileChunk piece, long responseLength, Uri uri, EventfulConcurrentQueue <FileChunk> asyncTasks) { Tuple <Task <HttpResponseMessage>, FileChunk> returnTuple; using (var wcObj = _wcPool.Get()) { Console.Title = "STREAMING...."; // Open a http request with the range var request = new HttpRequestMessage { RequestUri = uri }; request.Headers.ConnectionClose = false; request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End); // Send the request var downloadTask = wcObj.Value.SendAsync( request, HttpCompletionOption.ResponseContentRead, CancellationToken.None); // Use interlocked to increment Tasks done by one Interlocked.Add(ref TasksDone, 1); asyncTasks.Enqueue(piece); returnTuple = new Tuple <Task <HttpResponseMessage>, FileChunk>(downloadTask, piece); } return(returnTuple); }
public async Task DownloadByteArray(string URL, double parts, Action <byte[]> OnComplete, Action <int> OnUpdate = null) { var responseLength = (await WebRequest.Create(URL).GetResponseAsync()).ContentLength; var partSize = (long)Math.Floor(responseLength / parts); var pieces = new List <FileChunk>(); ThreadPool.GetMaxThreads(out int maxworkerThreads, out int maxconcurrentActiveRequests); bool changeSucceeded = ThreadPool.SetMaxThreads(maxworkerThreads, maxconcurrentActiveRequests); Console.WriteLine(responseLength.ToString(CultureInfo.InvariantCulture) + " TOTAL SIZE"); Console.WriteLine(partSize.ToString(CultureInfo.InvariantCulture) + " PART SIZE" + "\n"); try { // Using our custom progressbar using (var progress = new ProgressBar()) { using (var ms = new MemoryStream()) { ms.SetLength(responseLength); // Using custom concurrent queue to implement Enqueue and Dequeue // Events var asyncTasks = new EventfulConcurrentQueue <FileChunk>(); // Delegate for Dequeue asyncTasks.ItemDequeued += delegate { // Tasks done holds the count of the tasks done // Parts *2 because there are Parts number of Enqueue AND Dequeue // operations if (OnUpdate == null) { progress.Report(TasksDone / (parts * 2)); Thread.Sleep(20); } else { OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; // Delegate for Enqueue asyncTasks.ItemEnqueued += delegate { if (OnUpdate == null) { progress.Report(TasksDone / (parts * 2)); Thread.Sleep(20); } else { OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2))); } }; // GetResponseAsync deadlocks for some reason so switched to // HttpClient instead var client = new HttpClient( // Use our custom Retry handler, with a max retry value of 10 new RetryHandler(new HttpClientHandler(), 10)) { MaxResponseContentBufferSize = 1000000000 }; client.DefaultRequestHeaders.ConnectionClose = false; client.Timeout = Timeout.InfiniteTimeSpan; // Variable to hold the old loop end var previous = 0; // Loop to add all the events to the queue for (var i = (int)partSize; i <= responseLength; i += (int)partSize) { Console.Title = "WRITING CHUNKS..."; if (i + partSize < responseLength) { // Start and end values for the chunk var start = previous; var currentEnd = i; pieces.Add(new FileChunk(start, currentEnd)); // Set the start of the next loop to be the current end previous = currentEnd; } else { // Start and end values for the chunk var start = previous; var currentEnd = i; pieces.Add(new FileChunk(start, (int)responseLength)); // Set the start of the next loop to be the current end previous = currentEnd; } } var getFileChunk = new TransformManyBlock <IEnumerable <FileChunk>, FileChunk>( chunk => chunk, new ExecutionDataflowBlockOptions() { BoundedCapacity = Int32.MaxValue, // Cap the item count MaxDegreeOfParallelism = Environment .ProcessorCount, // Parallelize on all cores }); var getStream = new TransformBlock < FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> >( piece => { Console.Title = "STREAMING...."; // Open a http request with the range var request = new HttpRequestMessage { RequestUri = new Uri(URL) }; request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End); // Send the request var downloadTask = client.SendAsync( request, HttpCompletionOption.ResponseContentRead); // Use interlocked to increment Tasks done by one Interlocked.Add(ref TasksDone, 1); asyncTasks.Enqueue(piece); return(new Tuple <Task <HttpResponseMessage>, FileChunk>( downloadTask, piece)); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount, // Parallelize on all cores }); var writeStream = new ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> >( async tuple => { var buffer = new byte[tuple.Item2.End - tuple.Item2.Start]; using (var stream = await tuple.Item1.Result.Content .ReadAsStreamAsync()) { await stream.ReadAsync(buffer, 0, buffer.Length); } lock (ms) { ms.Position = tuple.Item2.Start; ms.Write(buffer, 0, buffer.Length); } var s = new FileChunk(); asyncTasks.TryDequeue(out s); Interlocked.Add(ref TasksDone, 1); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment .ProcessorCount, // Parallelize on all cores }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; getFileChunk.LinkTo(getStream, linkOptions); getStream.LinkTo(writeStream, linkOptions); getFileChunk.Post(pieces); getFileChunk.Complete(); await writeStream.Completion.ContinueWith(task => { if (asyncTasks.Count == 0) { ms.Flush(); ms.Close(); OnComplete?.Invoke(ms.ToArray()); } }); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } }