public static RequestFailedException ClientRequestIdMismatch(Response response, string echo, string original) => StorageExceptionExtensions.CreateException( response, $"Response x-ms-client-request-id '{echo}' does not match the original expected request id, '{original}'.");
/// <summary> /// Given <paramref name="ranges"/>, download content and write it to <paramref name="destinationStream"/>. /// </summary> /// <typeparam name="P"> /// Response type when downloading a single partition. /// </typeparam> /// <param name="destinationStream"> /// The stream to write content into. /// </param> /// <param name="etag"> /// The ETag of the content, for concurrency detection. /// </param> /// <param name="ranges"> /// The ordered set of ranges to download. /// </param> /// <param name="downloadPartitionAsync"> /// Returns a Task that will download a single partition of a stream (given the /// partition's stream, sequence number, whether to execute it /// async, and a cancellation token). /// </param> /// <param name="writePartitionAsync"> /// Returns a Task that writes the content stream into the destination stream (given the /// download response return by <paramref name="downloadPartitionAsync"/> and /// <paramref name="destinationStream"/>). /// </param> /// <param name="maximumActivePartitionCount"> /// The maximum number of partitions to download in parallel. /// </param> /// <param name="maximumLoadedPartitionCount"> /// The maximum number of partitions to retain in memory. /// </param> /// <param name="async"> /// Whether to perform the download asynchronously. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns></returns> /// <remarks> /// This method assumes that individual downloads are automatically retried. /// </remarks> private static async Task DownloadRangesImplAsync <P>( Stream destinationStream, ETag etag, IEnumerable <HttpRange> ranges, Func <ETag, HttpRange, bool, CancellationToken, Task <Response <P> > > downloadPartitionAsync, Func <Response <P>, Stream, bool, CancellationToken, Task> writePartitionAsync, int maximumActivePartitionCount, int maximumLoadedPartitionCount, bool async, CancellationToken cancellationToken) { // Use a queue to accumulate download tasks and return them in FIFO order. // Not using a ConcurrentQueue, since we aren't going to write using multiple threads here. // Based on prior research, we are better off just downloading the ranges, and writing them in order: // - Required for a non-seekable destination stream; // - Better performance than a MemoryMappedViewStream, because the file system won't have to zero out // spans skipped during random writes; // - Not necessarily as performant as an in-memory seekable stream, but memory streams probably aren't in the // size range where parallel download is really going to be useful anyway. // // We will still download in parallel, but limit ourselves to a maximum number of responses retained in memory, // and only await the head of the queue. var activeTaskQueue = new Queue <Task <Response <P> > >(); var loadedResponseQueue = new Queue <Response <P> >(); IEnumerator <HttpRange> rangesEnumerator = ranges.GetEnumerator(); while (true) { // Keep the queues filled. We could be more interesting with background threads and various semaphores or locks, // but this should be good-enough for the download case, given the ordering restriction. while (activeTaskQueue.Any() && activeTaskQueue.Peek().Status != TaskStatus.Running) { cancellationToken.ThrowIfCancellationRequested(); Task <Response <P> > responseTask = activeTaskQueue.Dequeue(); Response <P> response = async ? await responseTask.ConfigureAwait(false) : responseTask.EnsureCompleted(); loadedResponseQueue.Enqueue(response); } while ( activeTaskQueue.Count < maximumActivePartitionCount && (activeTaskQueue.Count + loadedResponseQueue.Count < maximumLoadedPartitionCount) ) { if (!rangesEnumerator.MoveNext()) { break; } HttpRange currentRange = rangesEnumerator.Current; cancellationToken.ThrowIfCancellationRequested(); Task <Task <Response <P> > > newTask = Task.Factory.StartNew( async() => await downloadPartitionAsync(etag, currentRange, async, cancellationToken).ConfigureAwait(false), cancellationToken, TaskCreationOptions.None, TaskScheduler.Default ); activeTaskQueue.Enqueue(newTask.Unwrap()); } if (loadedResponseQueue.Any()) { // await the completion of the head task, then write it to the destination cancellationToken.ThrowIfCancellationRequested(); Response <P> response = loadedResponseQueue.Dequeue(); // There's nothing to write for 304s if (response.IsUnavailable()) { // Turn the exploding response into a real exception // since we can't complete the download Response raw = response.GetRawResponse(); throw StorageExceptionExtensions.CreateException( raw, raw.ReasonPhrase, null, raw.Headers.TryGetValue(Constants.HeaderNames.ErrorCode, out string code) ? code : null); } Task writePartitionTask = writePartitionAsync(response, destinationStream, async, cancellationToken); if (async) { await writePartitionTask.ConfigureAwait(false); } else { writePartitionTask.EnsureCompleted(); } response.GetRawResponse().Dispose(); } else if (!activeTaskQueue.Any()) { // all downloads are completed break; } } }