getStreamTask(FileChunk piece, long responseLength, Uri uri,
EventfulConcurrentQueue <FileChunk> asyncTasks)
{
Tuple <Task <HttpResponseMessage>, FileChunk> returnTuple;
using (var wcObj = _wcPool.Get()) {
Console.Title = "STREAMING....";
// Open a http request with the range
var request = new HttpRequestMessage {
RequestUri = uri
};
request.Headers.ConnectionClose = false;
request.Headers.Range = new RangeHeaderValue(piece.Start, piece.End);
// Send the request
var downloadTask = wcObj.Value.SendAsync(
request, HttpCompletionOption.ResponseContentRead,
CancellationToken.None);
// Use interlocked to increment Tasks done by one
Interlocked.Add(ref TasksDone, 1);
asyncTasks.Enqueue(piece);
returnTuple =
new Tuple <Task <HttpResponseMessage>, FileChunk>(downloadTask, piece);
}
return(returnTuple);
}
public async Task DownloadByteArray(string URL, double parts,
Action <byte[]> OnComplete,
Action <int> OnUpdate = null)
{
var responseLength =
(await WebRequest.Create(URL).GetResponseAsync()).ContentLength;
var partSize = (long)Math.Floor(responseLength / parts);
var pieces = new List <FileChunk>();
ThreadPool.GetMaxThreads(out int maxworkerThreads,
out int maxconcurrentActiveRequests);
bool changeSucceeded =
ThreadPool.SetMaxThreads(maxworkerThreads, maxconcurrentActiveRequests);
Console.WriteLine(responseLength.ToString(CultureInfo.InvariantCulture) +
" TOTAL SIZE");
Console.WriteLine(partSize.ToString(CultureInfo.InvariantCulture) +
" PART SIZE" + "\n");
try {
// Using our custom progressbar
using (var progress = new ProgressBar()) {
using (var ms = new MemoryStream()) {
ms.SetLength(responseLength);
// Using custom concurrent queue to implement Enqueue and Dequeue
// Events
var asyncTasks = new EventfulConcurrentQueue <FileChunk>();
// Delegate for Dequeue
asyncTasks.ItemDequeued += delegate {
// Tasks done holds the count of the tasks done
// Parts *2 because there are Parts number of Enqueue AND Dequeue
// operations
if (OnUpdate == null)
{
progress.Report(TasksDone / (parts * 2));
Thread.Sleep(20);
}
else
{
OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2)));
}
};
// Delegate for Enqueue
asyncTasks.ItemEnqueued += delegate {
if (OnUpdate == null)
{
progress.Report(TasksDone / (parts * 2));
Thread.Sleep(20);
}
else
{
OnUpdate?.Invoke(Convert.ToInt32(TasksDone / (parts * 2)));
}
};
// GetResponseAsync deadlocks for some reason so switched to
// HttpClient instead
var client = new HttpClient(
// Use our custom Retry handler, with a max retry value of 10
new RetryHandler(new HttpClientHandler(), 10))
{
MaxResponseContentBufferSize = 1000000000
};
client.DefaultRequestHeaders.ConnectionClose = false;
client.Timeout = Timeout.InfiniteTimeSpan;
// Variable to hold the old loop end
var previous = 0;
// Loop to add all the events to the queue
for (var i = (int)partSize; i <= responseLength;
i += (int)partSize)
{
Console.Title = "WRITING CHUNKS...";
if (i + partSize < responseLength)
{
// Start and end values for the chunk
var start = previous;
var currentEnd = i;
pieces.Add(new FileChunk(start, currentEnd));
// Set the start of the next loop to be the current end
previous = currentEnd;
}
else
{
// Start and end values for the chunk
var start = previous;
var currentEnd = i;
pieces.Add(new FileChunk(start, (int)responseLength));
// Set the start of the next loop to be the current end
previous = currentEnd;
}
}
var getFileChunk =
new TransformManyBlock <IEnumerable <FileChunk>, FileChunk>(
chunk => chunk,
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = Int32.MaxValue, // Cap the item count
MaxDegreeOfParallelism =
Environment
.ProcessorCount, // Parallelize on all cores
});
var getStream = new TransformBlock <
FileChunk, Tuple <Task <HttpResponseMessage>, FileChunk> >(
piece => {
Console.Title = "STREAMING....";
// Open a http request with the range
var request = new HttpRequestMessage {
RequestUri = new Uri(URL)
};
request.Headers.Range =
new RangeHeaderValue(piece.Start, piece.End);
// Send the request
var downloadTask = client.SendAsync(
request, HttpCompletionOption.ResponseContentRead);
// Use interlocked to increment Tasks done by one
Interlocked.Add(ref TasksDone, 1);
asyncTasks.Enqueue(piece);
return(new Tuple <Task <HttpResponseMessage>, FileChunk>(
downloadTask, piece));
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = (int)parts, // Cap the item count
MaxDegreeOfParallelism =
Environment.ProcessorCount, // Parallelize on all cores
});
var writeStream =
new ActionBlock <Tuple <Task <HttpResponseMessage>, FileChunk> >(
async tuple => {
var buffer = new byte[tuple.Item2.End - tuple.Item2.Start];
using (var stream = await tuple.Item1.Result.Content
.ReadAsStreamAsync()) {
await stream.ReadAsync(buffer, 0, buffer.Length);
}
lock (ms) {
ms.Position = tuple.Item2.Start;
ms.Write(buffer, 0, buffer.Length);
}
var s = new FileChunk();
asyncTasks.TryDequeue(out s);
Interlocked.Add(ref TasksDone, 1);
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = (int)parts, // Cap the item count
MaxDegreeOfParallelism =
Environment
.ProcessorCount, // Parallelize on all cores
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
getFileChunk.LinkTo(getStream, linkOptions);
getStream.LinkTo(writeStream, linkOptions);
getFileChunk.Post(pieces);
getFileChunk.Complete();
await writeStream.Completion.ContinueWith(task => {
if (asyncTasks.Count == 0)
{
ms.Flush();
ms.Close();
OnComplete?.Invoke(ms.ToArray());
}
});
}
}
} catch (Exception ex) {
Console.WriteLine(ex.Message);
}
}
