static async Task Run(string connectionString, Options options)
{
#if DEBUG
if (!options.Debug)
{
throw new InvalidOperationException("Requires release configuration");
}
#endif
var httpClientHandler = new HttpClientHandler();
httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true;
var httpClient = new HttpClient(httpClientHandler);
var blobClientOptions = new BlobClientOptions();
blobClientOptions.Transport = new HttpClientTransport(httpClient);
var client = new BlobClient(connectionString, _containerName, _blobName, blobClientOptions);
if (options.Upload)
{
await UploadAndVerifyDownload(client, options.Size);
}
var parallelTransferOptions = new ParallelTransferOptions();
if (options.MaximumThreadCount != -1)
{
parallelTransferOptions.MaximumThreadCount = options.MaximumThreadCount;
}
Console.WriteLine($"Downloading blob '{_containerName}/{_blobName}' with {options.Parallel} parallel task(s) for {options.Duration} second(s)...");
Console.WriteLine();
var duration = TimeSpan.FromSeconds(options.Duration);
var cts = new CancellationTokenSource(duration);
var token = cts.Token;
var tasks = new Task[options.Parallel];
var sw = Stopwatch.StartNew();
for (var i = 0; i < options.Parallel; i++)
{
tasks[i] = DownloadLoop(client, parallelTransferOptions, token);
}
_ = PrintStatus(token);
await Task.WhenAll(tasks);
sw.Stop();
var elapsedSeconds = sw.Elapsed.TotalSeconds;
var downloadsPerSecond = _downloads / elapsedSeconds;
var megabytesPerSecond = (downloadsPerSecond * options.Size) / (1024 * 1024);
Console.WriteLine();
Console.WriteLine($"Downloaded {_downloads} blobs of size {options.Size} in {elapsedSeconds:N2}s " +
$"({downloadsPerSecond:N2} blobs/s, {megabytesPerSecond:N2} MB/s)");
}
static async Task DownloadLoop(BlobClient client, ParallelTransferOptions parallelTransferOptions, CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
await client.DownloadAsync(Stream.Null, parallelTransferOptions : parallelTransferOptions, cancellationToken : token);
Interlocked.Increment(ref _downloads);
}
catch (OperationCanceledException)
{
}
}
}
public virtual Task <Response <BlobContentInfo> > UploadAsync(
FileInfo content,
BlobHttpHeaders?blobHttpHeaders = default,
Metadata metadata = default,
BlobAccessConditions?blobAccessConditions = default,
IProgress <StorageProgress> progressHandler = default,
ParallelTransferOptions parallelTransferOptions = default,
CancellationToken cancellationToken = default) =>
this.StagedUploadAsync(
content,
blobHttpHeaders,
metadata,
blobAccessConditions,
progressHandler,
parallelTransferOptions: parallelTransferOptions,
async: true,
cancellationToken: cancellationToken);
public virtual Response <BlobContentInfo> Upload(
Stream content,
BlobHttpHeaders?blobHttpHeaders = default,
Metadata metadata = default,
BlobAccessConditions?blobAccessConditions = default,
IProgress <StorageProgress> progressHandler = default,
ParallelTransferOptions parallelTransferOptions = default,
CancellationToken cancellationToken = default) =>
this.StagedUploadAsync(
content,
blobHttpHeaders,
metadata,
blobAccessConditions,
progressHandler,
parallelTransferOptions: parallelTransferOptions,
async: false,
cancellationToken: cancellationToken)
.EnsureCompleted();
/// <summary>
/// This operation will create a new
/// block blob of arbitrary size by uploading it as indiviually staged
/// blocks if it's larger than the
/// <paramref name="singleBlockThreshold"/>.
/// </summary>
/// <param name="file">
/// A <see cref="FileInfo"/> of the file to upload.
/// </param>
/// <param name="blobHttpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// block blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this block blob.
/// </param>
/// <param name="blobAccessConditions">
/// Optional <see cref="BlobAccessConditions"/> to add conditions on
/// the creation of this new block blob.
/// </param>
/// <param name="progressHandler">
/// Optional <see cref="IProgress{StorageProgress}"/> to provide
/// progress updates about data transfers.
/// </param>
/// <param name="singleBlockThreshold">
/// The maximum size stream that we'll upload as a single block. The
/// default value is 256MB.
/// </param>
/// <param name="parallelTransferOptions">
/// Optional <see cref="ParallelTransferOptions"/> to configure
/// parallel transfer behavior.
/// </param>
/// <param name="async">
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="StorageRequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
FileInfo file,
BlobHttpHeaders?blobHttpHeaders,
Metadata metadata,
BlobAccessConditions?blobAccessConditions,
IProgress <StorageProgress> progressHandler,
long singleBlockThreshold = BlockBlobClient.BlockBlobMaxUploadBlobBytes,
ParallelTransferOptions parallelTransferOptions = default,
bool async = true,
CancellationToken cancellationToken = default)
{
Debug.Assert(singleBlockThreshold <= BlockBlobClient.BlockBlobMaxUploadBlobBytes);
var client = new BlockBlobClient(this.Uri, this.Pipeline);
var blockMap = new ConcurrentDictionary <long, string>();
var blockName = 0;
var uploadTask = PartitionedUploader.UploadAsync(
UploadStreamAsync,
StageBlockAsync,
CommitBlockListAsync,
threshold => file.Length < threshold,
memoryPool => new StreamPartitioner(file, memoryPool),
singleBlockThreshold,
parallelTransferOptions,
async,
cancellationToken);
return(async ?
await uploadTask.ConfigureAwait(false) :
uploadTask.EnsureCompleted());
string GetNewBase64BlockId(long blockOrdinal)
{
// Create and record a new block ID, storing the order information
// (nominally the block's start position in the original stream)
var newBlockName = Interlocked.Increment(ref blockName);
var blockId = Constants.BlockNameFormat;
blockId = String.Format(CultureInfo.InvariantCulture, blockId, newBlockName);
blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));
var success = blockMap.TryAdd(blockOrdinal, blockId);
Debug.Assert(success);
return(blockId);
}
// Upload the entire stream
async Task <Response <BlobContentInfo> > UploadStreamAsync()
{
using (var stream = file.OpenRead())
{
return
(await client.UploadInternal(
stream,
blobHttpHeaders,
metadata,
blobAccessConditions,
progressHandler,
async,
cancellationToken)
.ConfigureAwait(false));
}
}
// Upload a single partition of the stream
Task <Response <BlockInfo> > StageBlockAsync(
Stream partition,
long blockOrdinal,
bool async,
CancellationToken cancellation)
{
var base64BlockId = GetNewBase64BlockId(blockOrdinal);
//var bytes = new byte[10];
//partition.Read(bytes, 0, 10);
partition.Position = 0;
//Console.WriteLine($"Commiting partition {blockOrdinal} => {base64BlockId}, {String.Join(" ", bytes)}");
// Upload the block
return(client.StageBlockInternal(
base64BlockId,
partition,
null,
blobAccessConditions?.LeaseAccessConditions,
progressHandler,
async,
cancellationToken));
}
// Commit a series of partitions
Task <Response <BlobContentInfo> > CommitBlockListAsync(
bool async,
CancellationToken cancellation)
{
var base64BlockIds = blockMap.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray();
//Console.WriteLine($"Commiting block list:\n{String.Join("\n", base64BlockIds)}");
return
(client.CommitBlockListInternal(
base64BlockIds,
blobHttpHeaders,
metadata,
blobAccessConditions,
async,
cancellationToken));
}
}
static async Task Run(string connectionString, Options options)
{
#if DEBUG
if (!options.Debug)
{
throw new InvalidOperationException("Requires release configuration");
}
#endif
var httpClientHandler = new HttpClientHandler();
httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true;
var httpClient = new HttpClient(httpClientHandler);
var blobClientOptions = new BlobClientOptions();
blobClientOptions.Transport = new HttpClientTransport(httpClient);
var blockClient = new BlockBlobClient(connectionString, _containerName, _blobName, blobClientOptions);
var blockList = (await blockClient.GetBlockListAsync()).Value;
PrintBlockList(blockList);
var client = new BlobClient(connectionString, _containerName, _blobName, blobClientOptions);
var parallelTransferOptions = new ParallelTransferOptions()
{
MaximumThreadCount = options.MaximumThreadCount,
MaximumTransferLength = options.MaximumTransferLength
};
var payload = new byte[options.Size];
// Initialize payload with stable random data since all-zeros may be compressed or optimized
(new Random(0)).NextBytes(payload);
var payloadStream = new MemoryStream(payload, writable: false);
Console.WriteLine($"Uploading and downloading blob of size {options.Size} with {options.MaximumThreadCount} threads...");
Console.WriteLine();
var sw = new Stopwatch();
for (var i = 0; i < options.Count; i++)
{
payloadStream.Seek(0, SeekOrigin.Begin);
sw.Restart();
await client.UploadAsync(payloadStream, parallelTransferOptions : parallelTransferOptions);
sw.Stop();
var elapsedSeconds = sw.Elapsed.TotalSeconds;
var megabytesPerSecond = (options.Size / (1024 * 1024)) / elapsedSeconds;
Console.WriteLine($"Uploaded {options.Size} bytes in {elapsedSeconds:N2} seconds ({megabytesPerSecond:N2} MB/s)");
sw.Restart();
await client.DownloadAsync(Stream.Null, parallelTransferOptions : parallelTransferOptions);
sw.Stop();
elapsedSeconds = sw.Elapsed.TotalSeconds;
megabytesPerSecond = (options.Size / (1024 * 1024)) / elapsedSeconds;
Console.WriteLine($"Downloaded {options.Size} bytes in {elapsedSeconds:N2} seconds ({megabytesPerSecond:N2} MB/s)");
Console.WriteLine();
}
}
