public Response <PathInfo> Upload(
Stream content,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
_client.Create(
httpHeaders: httpHeaders,
conditions: conditions,
cancellationToken: cancellationToken);
// After the File is Create, Lease ID is the only valid request parameter.
conditions = new DataLakeRequestConditions {
LeaseId = conditions?.LeaseId
};
// If we can compute the size and it's small enough
if (PartitionedUploadExtensions.TryGetLength(content, out long contentLength) &&
contentLength < _singleUploadThreshold)
{
// Upload it in a single request
_client.Append(
content,
offset: 0,
leaseId: conditions?.LeaseId,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
// Calculate flush position
long flushPosition = contentLength;
return(_client.Flush(
position: flushPosition,
httpHeaders: httpHeaders,
conditions: conditions,
cancellationToken: cancellationToken));
}
// If the caller provided an explicit block size, we'll use it.
// Otherwise we'll adjust dynamically based on the size of the
// content.
int blockSize =
_blockSize != null ? _blockSize.Value :
contentLength < Constants.LargeUploadThreshold ?
Constants.DefaultBufferSize :
Constants.LargeBufferSize;
// Otherwise stage individual blocks one at a time. It's not as
// fast as a parallel upload, but you get the benefit of the retry
// policy working on a single block instead of the entire stream.
return(UploadInSequence(
content,
blockSize,
httpHeaders,
conditions,
progressHandler,
cancellationToken));
}
public async Task <Response <PathInfo> > UploadAsync(
Stream content,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
await _client.CreateAsync(
httpHeaders : httpHeaders,
conditions : conditions,
cancellationToken : cancellationToken).ConfigureAwait(false);
// After the File is Create, Lease ID is the only valid request parameter.
conditions = new DataLakeRequestConditions {
LeaseId = conditions?.LeaseId
};
// If we can compute the size and it's small enough
if (PartitionedUploadExtensions.TryGetLength(content, out long contentLength) &&
contentLength < _singleUploadThreshold)
{
// Append data
await _client.AppendAsync(
content,
offset : 0,
leaseId : conditions?.LeaseId,
progressHandler : progressHandler,
cancellationToken : cancellationToken).ConfigureAwait(false);
// Flush data
return(await _client.FlushAsync(
position : contentLength,
httpHeaders : httpHeaders,
conditions : conditions)
.ConfigureAwait(false));
}
// If the caller provided an explicit block size, we'll use it.
// Otherwise we'll adjust dynamically based on the size of the
// content.
int blockSize =
_blockSize != null ? _blockSize.Value :
contentLength < Constants.LargeUploadThreshold ?
Constants.DefaultBufferSize :
Constants.LargeBufferSize;
// Otherwise stage individual blocks in parallel
return(await UploadInParallelAsync(
content,
blockSize,
httpHeaders,
conditions,
progressHandler,
cancellationToken).ConfigureAwait(false));
}
public async Task <Response <BlobContentInfo> > UploadAsync(
Stream content,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier = default,
CancellationToken cancellationToken = default)
{
// If we can compute the size and it's small enough
if (PartitionedUploadExtensions.TryGetLength(content, out long length) && length < _singleUploadThreshold)
{
// Upload it in a single request
return(await _client.UploadInternal(
content,
blobHttpHeaders,
metadata,
conditions,
accessTier,
progressHandler,
_operationName,
async : true,
cancellationToken)
.ConfigureAwait(false));
}
// If the caller provided an explicit block size, we'll use it.
// Otherwise we'll adjust dynamically based on the size of the
// content.
int blockSize =
_blockSize != null ? _blockSize.Value :
length < Constants.LargeUploadThreshold ?
Constants.DefaultBufferSize :
Constants.LargeBufferSize;
// Otherwise stage individual blocks in parallel
return(await UploadInParallelAsync(
content,
blockSize,
blobHttpHeaders,
metadata,
conditions,
progressHandler,
accessTier,
cancellationToken)
.ConfigureAwait(false));
}
public Response <BlobContentInfo> Upload(
Stream content,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier = default,
CancellationToken cancellationToken = default)
{
// If we can compute the size and it's small enough
if (PartitionedUploadExtensions.TryGetLength(content, out long length) && length < _singleUploadThreshold)
{
// Upload it in a single request
return(_client.UploadInternal(
content,
blobHttpHeaders,
metadata,
conditions,
accessTier,
progressHandler,
_operationName,
false,
cancellationToken).EnsureCompleted());
}
// If the caller provided an explicit block size, we'll use it.
// Otherwise we'll adjust dynamically based on the size of the
// content.
int blockSize =
_blockSize != null ? _blockSize.Value :
length < Constants.LargeUploadThreshold ?
Constants.DefaultBufferSize :
Constants.LargeBufferSize;
// Otherwise stage individual blocks one at a time. It's not as
// fast as a parallel upload, but you get the benefit of the retry
// policy working on a single block instead of the entire stream.
return(UploadInSequence(
content,
blockSize,
blobHttpHeaders,
metadata,
conditions,
progressHandler,
accessTier,
cancellationToken));
}
private async Task <Response <PathInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Files)}.{nameof(DataLake)}.{nameof(DataLakeFileClient)}.{nameof(DataLakeFileClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// We need to keep track of how much data we have appended to
// calculate offsets for the next appends, and the final
// position to flush
long appendedBytes = 0;
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content, blockSize, async: true, _arrayPool, cancellationToken).ConfigureAwait(false))
{
// Start appending the next block (but don't await the Task!)
Task task = AppendBlockAsync(
block,
appendedBytes,
conditions?.LeaseId,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
appendedBytes += block.Length;
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.FlushAsync(
position : appendedBytes,
httpHeaders : httpHeaders,
conditions : conditions,
cancellationToken : cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private Response <PathInfo> UploadInSequence(
Stream content,
int blockSize,
PathHttpHeaders httpHeaders,
DataLakeRequestConditions conditions,
IProgress <long> progressHandler,
CancellationToken cancellationToken)
{
// Wrap the append and flush calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Files)}.{nameof(DataLake)}.{nameof(DataLakeFileClient)}.{nameof(DataLakeFileClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each append file operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// Partition the stream into individual blocks and stage them
// We need to keep track of how much data we have appended to
// calculate offsets for the next appends, and the final
// position to flush
long appendedBytes = 0;
foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content, blockSize, async: false, _arrayPool, cancellationToken).EnsureSyncEnumerable())
{
// Dispose the block after the loop iterates and return its memory to our ArrayPool
using (block)
{
// Append the next block
_client.Append(
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
offset: appendedBytes,
leaseId: conditions?.LeaseId,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
appendedBytes += block.Length;
}
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.Flush(
position: appendedBytes,
httpHeaders: httpHeaders,
conditions: conditions,
cancellationToken: cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private async Task <Response <BlobContentInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content,
blockSize,
async: true,
_arrayPool,
cancellationToken).ConfigureAwait(false))
{
// Start staging the next block (but don't await the Task!)
string blockId = GenerateBlockId(block.AbsolutePosition);
Task task = StageBlockAsync(
block,
blockId,
conditions,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
blockIds.Add(blockId);
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.CommitBlockListAsync(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private Response <BlobContentInfo> UploadInSequence(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// Partition the stream into individual blocks and stage them
foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content, blockSize, async: false, _arrayPool, cancellationToken).EnsureSyncEnumerable())
{
// Dispose the block after the loop iterates and return its memory to our ArrayPool
using (block)
{
// Stage the next block
string blockId = GenerateBlockId(block.AbsolutePosition);
_client.StageBlock(
blockId,
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
conditions: conditions,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
blockIds.Add(blockId);
}
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.CommitBlockList(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
