private Tuple <Task <HttpResponseMessage>, FileChunk> GetStreamTask(FileChunk piece, long responseLength, Uri uri, EventfulConcurrentQueue <FileChunk> asyncTasks) { using var wcObj = _wcPool.Get(); Progress.Detail = "正在下载……"; //Open a http request with the range HttpRequestMessage request = new HttpRequestMessage { RequestUri = uri }; request.Headers.ConnectionClose = false; request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End); //Send the request var downloadTask = wcObj.Value.SendAsync(request, HttpCompletionOption.ResponseContentRead, CancellationToken.None); //Use interlocked to increment Tasks done by one Interlocked.Add(ref _tasksDone, 1); asyncTasks.Enqueue(piece); return(new Tuple <Task <HttpResponseMessage>, FileChunk>(downloadTask, piece)); }
public async Task <byte[]> DownloadByteArray(string url, double parts) { _responseLength = (await WebRequest.Create(url).GetResponseAsync()).ContentLength; long partSize = (long)Math.Floor(_responseLength / parts); var pieces = new List <FileChunk>(); ThreadPool.GetMaxThreads(out int maxWorkerThreads, out int maxConcurrentActiveRequests); bool changeSucceeded = ThreadPool.SetMaxThreads( maxWorkerThreads, maxConcurrentActiveRequests); //Console.WriteLine(responseLength + " TOTAL SIZE"); //Console.WriteLine(partSize + " PART SIZE" + "\n"); Progress.Detail = $"文件总大小:{_responseLength}字节"; try { using MemoryStream ms = new MemoryStream(); ms.SetLength(_responseLength); //Using custom concurrent queue to implement Enqueue and Dequeue Events var asyncTasks = new EventfulConcurrentQueue <FileChunk>(); //Delegate for Dequeue asyncTasks.ItemDequeued += delegate { //Tasks done holds the count of the tasks done //Parts *2 because there are Parts number of Enqueue AND Dequeue operations Progress.Percentage = _tasksDone / (parts * 2); }; //Delegate for Enqueue asyncTasks.ItemEnqueued += delegate { Progress.Percentage = _tasksDone / (parts * 2); }; // GetResponseAsync deadlocks for some reason so switched to HttpClient instead HttpClient client = new HttpClient( //Use our custom Retry handler, with a max retry value of 10 new RetryHandler(new HttpClientHandler(), 10)) { MaxResponseContentBufferSize = 1000000000 }; client.DefaultRequestHeaders.ConnectionClose = false; client.Timeout = Timeout.InfiniteTimeSpan; //Variable to hold the old loop end int previous = 0; //Loop to add all the events to the queue for (int i = (int)partSize; i <= _responseLength; i += (int)partSize) { Progress.Detail = "写入缓存……"; if (i + partSize < _responseLength) { //Start and end values for the chunk int start = previous; int currentEnd = i; pieces.Add(new FileChunk(start, currentEnd)); //Set the start of the next loop to be the current end previous = currentEnd; } else { //Start and end values for the chunk int start = previous; int currentEnd = i; pieces.Add(new FileChunk(start, (int)_responseLength)); //Set the start of the next loop to be the current end previous = currentEnd; } } var getFileChunk = new TransformManyBlock <IEnumerable <FileChunk>, FileChunk>(chunk => chunk, new ExecutionDataflowBlockOptions { BoundedCapacity = int.MaxValue, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount // Parallelize on all cores }); var getStream = new TransformBlock <FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> >( piece => { Progress.Detail = "正在下载……"; //Open a http request with the range HttpRequestMessage request = new HttpRequestMessage { RequestUri = new Uri(url) }; request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End); //Send the request var downloadTask = client.SendAsync(request, HttpCompletionOption.ResponseContentRead); //Use interlocked to increment Tasks done by one Interlocked.Add(ref _tasksDone, 1); asyncTasks.Enqueue(piece); return(new Tuple <Task <HttpResponseMessage>, FileChunk>(downloadTask, piece)); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount // Parallelize on all cores } ); var writeStream = new ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> >(async tuple => { var buffer = new byte[tuple.Item2.End - tuple.Item2.Start]; using (Stream stream = await tuple.Item1.Result.Content.ReadAsStreamAsync()) { await stream.ReadAsync(buffer, 0, buffer.Length); } lock (ms) { ms.Position = tuple.Item2.Start; ms.Write(buffer, 0, buffer.Length); } FileChunk s = new FileChunk(); asyncTasks.TryDequeue(out s); Interlocked.Add(ref _tasksDone, 1); }, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)parts, // Cap the item count MaxDegreeOfParallelism = Environment.ProcessorCount // Parallelize on all cores }); DataflowLinkOptions linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; getFileChunk.LinkTo(getStream, linkOptions); getStream.LinkTo(writeStream, linkOptions); getFileChunk.Post(pieces); getFileChunk.Complete(); await writeStream.Completion.ContinueWith(task => { if (asyncTasks.Count != 0) { return; } ms.Flush(); ms.Close(); //onComplete?.Invoke(ms.ToArray()); }); Progress.TriggerComplete(this, EventArgs.Empty); return(ms.ToArray()); } catch (Exception ex) { Progress.Detail = "发生错误:" + ex.Message; } Progress.TriggerComplete(this, EventArgs.Empty); return(null); }