public static RequestFailedException ClientRequestIdMismatch(Response response, string echo, string original)
=> StorageExceptionExtensions.CreateException(
response,
$"Response x-ms-client-request-id '{echo}' does not match the original expected request id, '{original}'.");
/// <summary>
/// Given <paramref name="ranges"/>, download content and write it to <paramref name="destinationStream"/>.
/// </summary>
/// <typeparam name="P">
/// Response type when downloading a single partition.
/// </typeparam>
/// <param name="destinationStream">
/// The stream to write content into.
/// </param>
/// <param name="etag">
/// The ETag of the content, for concurrency detection.
/// </param>
/// <param name="ranges">
/// The ordered set of ranges to download.
/// </param>
/// <param name="downloadPartitionAsync">
/// Returns a Task that will download a single partition of a stream (given the
/// partition's stream, sequence number, whether to execute it
/// async, and a cancellation token).
/// </param>
/// <param name="writePartitionAsync">
/// Returns a Task that writes the content stream into the destination stream (given the
/// download response return by <paramref name="downloadPartitionAsync"/> and
/// <paramref name="destinationStream"/>).
/// </param>
/// <param name="maximumActivePartitionCount">
/// The maximum number of partitions to download in parallel.
/// </param>
/// <param name="maximumLoadedPartitionCount">
/// The maximum number of partitions to retain in memory.
/// </param>
/// <param name="async">
/// Whether to perform the download asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns></returns>
/// <remarks>
/// This method assumes that individual downloads are automatically retried.
/// </remarks>
private static async Task DownloadRangesImplAsync <P>(
Stream destinationStream,
ETag etag,
IEnumerable <HttpRange> ranges,
Func <ETag, HttpRange, bool, CancellationToken, Task <Response <P> > > downloadPartitionAsync,
Func <Response <P>, Stream, bool, CancellationToken, Task> writePartitionAsync,
int maximumActivePartitionCount,
int maximumLoadedPartitionCount,
bool async,
CancellationToken cancellationToken)
{
// Use a queue to accumulate download tasks and return them in FIFO order.
// Not using a ConcurrentQueue, since we aren't going to write using multiple threads here.
// Based on prior research, we are better off just downloading the ranges, and writing them in order:
// - Required for a non-seekable destination stream;
// - Better performance than a MemoryMappedViewStream, because the file system won't have to zero out
// spans skipped during random writes;
// - Not necessarily as performant as an in-memory seekable stream, but memory streams probably aren't in the
// size range where parallel download is really going to be useful anyway.
//
// We will still download in parallel, but limit ourselves to a maximum number of responses retained in memory,
// and only await the head of the queue.
var activeTaskQueue = new Queue <Task <Response <P> > >();
var loadedResponseQueue = new Queue <Response <P> >();
IEnumerator <HttpRange> rangesEnumerator = ranges.GetEnumerator();
while (true)
{
// Keep the queues filled. We could be more interesting with background threads and various semaphores or locks,
// but this should be good-enough for the download case, given the ordering restriction.
while (activeTaskQueue.Any() && activeTaskQueue.Peek().Status != TaskStatus.Running)
{
cancellationToken.ThrowIfCancellationRequested();
Task <Response <P> > responseTask = activeTaskQueue.Dequeue();
Response <P> response =
async
? await responseTask.ConfigureAwait(false)
: responseTask.EnsureCompleted();
loadedResponseQueue.Enqueue(response);
}
while (
activeTaskQueue.Count < maximumActivePartitionCount &&
(activeTaskQueue.Count + loadedResponseQueue.Count < maximumLoadedPartitionCount)
)
{
if (!rangesEnumerator.MoveNext())
{
break;
}
HttpRange currentRange = rangesEnumerator.Current;
cancellationToken.ThrowIfCancellationRequested();
Task <Task <Response <P> > > newTask = Task.Factory.StartNew(
async() => await downloadPartitionAsync(etag, currentRange, async, cancellationToken).ConfigureAwait(false),
cancellationToken,
TaskCreationOptions.None,
TaskScheduler.Default
);
activeTaskQueue.Enqueue(newTask.Unwrap());
}
if (loadedResponseQueue.Any())
{
// await the completion of the head task, then write it to the destination
cancellationToken.ThrowIfCancellationRequested();
Response <P> response = loadedResponseQueue.Dequeue();
// There's nothing to write for 304s
if (response.IsUnavailable())
{
// Turn the exploding response into a real exception
// since we can't complete the download
Response raw = response.GetRawResponse();
throw StorageExceptionExtensions.CreateException(
raw,
raw.ReasonPhrase,
null,
raw.Headers.TryGetValue(Constants.HeaderNames.ErrorCode, out string code) ? code : null);
}
Task writePartitionTask = writePartitionAsync(response, destinationStream, async, cancellationToken);
if (async)
{
await writePartitionTask.ConfigureAwait(false);
}
else
{
writePartitionTask.EnsureCompleted();
}
response.GetRawResponse().Dispose();
}
else if (!activeTaskQueue.Any())
{
// all downloads are completed
break;
}
}
}