private async Task <Response <BlobContentInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content,
blockSize,
async: true,
_arrayPool,
cancellationToken).ConfigureAwait(false))
{
// Start staging the next block (but don't await the Task!)
string blockId = GenerateBlockId(block.AbsolutePosition);
Task task = StageBlockAsync(
block,
blockId,
conditions,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
blockIds.Add(blockId);
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.CommitBlockListAsync(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
/// <summary>
/// The <see cref="StagedUploadAsync"/> operation will create a new
/// block blob of arbitrary size by uploading it as indiviually staged
/// blocks if it's larger than the
/// <paramref name="singleBlockThreshold"/>.
/// </summary>
/// <param name="content">
/// A <see cref="Stream"/> containing the content to upload.
/// </param>
/// <param name="blobHttpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// block blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this block blob.
/// </param>
/// <param name="blobAccessConditions">
/// Optional <see cref="BlobAccessConditions"/> to add conditions on
/// the creation of this new block blob.
/// </param>
/// <param name="progressHandler">
/// Optional <see cref="IProgress{StorageProgress}"/> to provide
/// progress updates about data transfers.
/// </param>
/// <param name="singleBlockThreshold">
/// The maximum size stream that we'll upload as a single block. The
/// default value is 256MB.
/// </param>
/// <param name="blockSize">
/// The size of individually staged blocks. The default value is 4MB.
/// </param>
/// <param name="async">
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Task{Response{BlobContentInfo}}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="StorageRequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
Stream content,
BlobHttpHeaders?blobHttpHeaders,
Metadata metadata,
BlobAccessConditions?blobAccessConditions,
IProgress <StorageProgress> progressHandler,
long singleBlockThreshold = BlockBlobClient.BlockBlobMaxUploadBlobBytes,
int blockSize = Constants.DefaultBufferSize,
bool async = true,
CancellationToken cancellationToken = default)
{
Debug.Assert(singleBlockThreshold <= BlockBlobClient.BlockBlobMaxUploadBlobBytes);
var client = new BlockBlobClient(this.Uri, this.Pipeline);
var blockList = new List <string>();
var uploadTask = ChunkedUploader.UploadAsync(
UploadStreamAsync,
StageBlockAsync,
CommitBlockListAsync,
content,
singleBlockThreshold,
blockSize,
async,
cancellationToken);
return(async ?
await uploadTask.ConfigureAwait(false) :
uploadTask.EnsureCompleted());
// Upload the entire stream
Task <Response <BlobContentInfo> > UploadStreamAsync(
Stream content,
bool async,
CancellationToken cancellation) =>
client.UploadAsync(
content,
blobHttpHeaders,
metadata,
blobAccessConditions,
progressHandler,
async,
cancellationToken);
// Upload a single chunk of the stream
Task <Response <BlockInfo> > StageBlockAsync(
Stream chunk,
int blockNumber,
bool async,
CancellationToken cancellation)
{
// Create a new block ID
var blockId = Constants.BlockNameFormat;
blockId = String.Format(CultureInfo.InvariantCulture, blockId, blockNumber);
blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));
blockList.Add(blockId);
// Upload the block
return(client.StageBlockAsync(
blockId,
chunk,
null,
blobAccessConditions?.LeaseAccessConditions,
progressHandler,
async,
cancellationToken));
}
// Commit a series of chunks
Task <Response <BlobContentInfo> > CommitBlockListAsync(
bool async,
CancellationToken cancellation) =>
client.CommitBlockListAsync(
blockList,
blobHttpHeaders,
metadata,
blobAccessConditions,
async,
cancellationToken);
}
