private async Task <Response <BlobContentInfo> > UploadInParallelAsync( Stream content, int blockSize, BlobHttpHeaders blobHttpHeaders, IDictionary <string, string> metadata, BlobRequestConditions conditions, IProgress <long> progressHandler, AccessTier?accessTier, CancellationToken cancellationToken) { // Wrap the staging and commit calls in an Upload span for // distributed tracing DiagnosticScope scope = _client.ClientDiagnostics.CreateScope( _operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}"); try { scope.Start(); // Wrap progressHandler in a AggregatingProgressIncrementer to prevent // progress from being reset with each stage blob operation. if (progressHandler != null) { progressHandler = new AggregatingProgressIncrementer(progressHandler); } // The list tracking blocks IDs we're going to commit List <string> blockIds = new List <string>(); // A list of tasks that are currently executing which will // always be smaller than _maxWorkerCount List <Task> runningTasks = new List <Task>(); // Partition the stream into individual blocks await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync( content, blockSize, async: true, _arrayPool, cancellationToken).ConfigureAwait(false)) { // Start staging the next block (but don't await the Task!) string blockId = GenerateBlockId(block.AbsolutePosition); Task task = StageBlockAsync( block, blockId, conditions, progressHandler, cancellationToken); // Add the block to our task and commit lists runningTasks.Add(task); blockIds.Add(blockId); // If we run out of workers if (runningTasks.Count >= _maxWorkerCount) { // Wait for at least one of them to finish await Task.WhenAny(runningTasks).ConfigureAwait(false); // Clear any completed blocks from the task list for (int i = 0; i < runningTasks.Count; i++) { Task runningTask = runningTasks[i]; if (!runningTask.IsCompleted) { continue; } await runningTask.ConfigureAwait(false); runningTasks.RemoveAt(i); i--; } } } // Wait for all the remaining blocks to finish staging and then // commit the block list to complete the upload await Task.WhenAll(runningTasks).ConfigureAwait(false); return(await _client.CommitBlockListAsync( blockIds, blobHttpHeaders, metadata, conditions, accessTier, cancellationToken) .ConfigureAwait(false)); } catch (Exception ex) { scope.Failed(ex); throw; } finally { scope.Dispose(); } }
/// <summary> /// The <see cref="StagedUploadAsync"/> operation will create a new /// block blob of arbitrary size by uploading it as indiviually staged /// blocks if it's larger than the /// <paramref name="singleBlockThreshold"/>. /// </summary> /// <param name="content"> /// A <see cref="Stream"/> containing the content to upload. /// </param> /// <param name="blobHttpHeaders"> /// Optional standard HTTP header properties that can be set for the /// block blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this block blob. /// </param> /// <param name="blobAccessConditions"> /// Optional <see cref="BlobAccessConditions"/> to add conditions on /// the creation of this new block blob. /// </param> /// <param name="progressHandler"> /// Optional <see cref="IProgress{StorageProgress}"/> to provide /// progress updates about data transfers. /// </param> /// <param name="singleBlockThreshold"> /// The maximum size stream that we'll upload as a single block. The /// default value is 256MB. /// </param> /// <param name="blockSize"> /// The size of individually staged blocks. The default value is 4MB. /// </param> /// <param name="async"> /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Task{Response{BlobContentInfo}}"/> describing the /// state of the updated block blob. /// </returns> /// <remarks> /// A <see cref="StorageRequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> internal async Task <Response <BlobContentInfo> > StagedUploadAsync( Stream content, BlobHttpHeaders?blobHttpHeaders, Metadata metadata, BlobAccessConditions?blobAccessConditions, IProgress <StorageProgress> progressHandler, long singleBlockThreshold = BlockBlobClient.BlockBlobMaxUploadBlobBytes, int blockSize = Constants.DefaultBufferSize, bool async = true, CancellationToken cancellationToken = default) { Debug.Assert(singleBlockThreshold <= BlockBlobClient.BlockBlobMaxUploadBlobBytes); var client = new BlockBlobClient(this.Uri, this.Pipeline); var blockList = new List <string>(); var uploadTask = ChunkedUploader.UploadAsync( UploadStreamAsync, StageBlockAsync, CommitBlockListAsync, content, singleBlockThreshold, blockSize, async, cancellationToken); return(async ? await uploadTask.ConfigureAwait(false) : uploadTask.EnsureCompleted()); // Upload the entire stream Task <Response <BlobContentInfo> > UploadStreamAsync( Stream content, bool async, CancellationToken cancellation) => client.UploadAsync( content, blobHttpHeaders, metadata, blobAccessConditions, progressHandler, async, cancellationToken); // Upload a single chunk of the stream Task <Response <BlockInfo> > StageBlockAsync( Stream chunk, int blockNumber, bool async, CancellationToken cancellation) { // Create a new block ID var blockId = Constants.BlockNameFormat; blockId = String.Format(CultureInfo.InvariantCulture, blockId, blockNumber); blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId)); blockList.Add(blockId); // Upload the block return(client.StageBlockAsync( blockId, chunk, null, blobAccessConditions?.LeaseAccessConditions, progressHandler, async, cancellationToken)); } // Commit a series of chunks Task <Response <BlobContentInfo> > CommitBlockListAsync( bool async, CancellationToken cancellation) => client.CommitBlockListAsync( blockList, blobHttpHeaders, metadata, blobAccessConditions, async, cancellationToken); }