/// <summary>
///
/// </summary>
/// <param name="sourceFileName"></param>
/// <param name="currRange"></param>
/// <param name="destBlob"></param>
/// <param name="logger"></param>
/// <returns></returns>
private async static Task <bool> ProcessBlockRange(
BlockRangeBase currRange,
CloudBlockBlob destBlob,
int timeoutSeconds,
bool useInbuiltRetry,
int retryCount,
bool calcMD5ForBlock,
ILogger logger)
{
logger.LogDebug($"Started ProcessBlockRange for {currRange.Name}");
using (var brData = await currRange.GetBlockRangeData(calcMD5ForBlock,
timeoutSeconds,
useInbuiltRetry,
retryCount,
logger))
{
var brDataIsNull = (brData is null) ? "null" : "valid";
logger.LogDebug($"GetBlockRangeData was called for block {currRange.Name} and returned brData {brDataIsNull}");
if (brData is null)
{
logger.LogDebug($"Returning false from GetBlockRangeData for block {currRange.Name} as brData was {brDataIsNull}");
return(false);
}
logger.LogDebug($"Inside ProcessBlockRange, about to start the PutBlockAsync action for {currRange.Name}");
// use retry policy which will automatically handle the throttling related StorageExceptions
await BlobHelpers.GetStorageRetryPolicy($"PutBlockAsync for block {currRange.Name}", retryCount, logger).ExecuteAsync(async() =>
{
logger.LogDebug($"Before PutBlockAsync for {currRange.Name}");
var blobReqOpts = new BlobRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromSeconds(timeoutSeconds)
};
if (!useInbuiltRetry)
{
blobReqOpts.RetryPolicy = new WindowsAzure.Storage.RetryPolicies.NoRetry();
}
// reset the memory stream again to 0
brData.MemStream.Position = 0;
// and then call Azure Storage to put this as a block with the given block ID and MD5 hash
await destBlob.PutBlockAsync(currRange.Name, brData.MemStream, brData.Base64EncodedMD5Checksum,
null, blobReqOpts, null);
logger.LogDebug($"Finished PutBlockAsync for {currRange.Name}");
});
logger.LogDebug($"Finished ProcessBlockRange for {currRange.Name}");
}
return(true);
}
internal async override Task <BlockRangeData> GetBlockRangeData(bool calcMD5ForBlock, int timeoutSeconds, bool useInbuiltRetry, int retryCount, ILogger logger)
{
BlockRangeData retVal = null;
// use retry policy which will automatically handle the throttling related StorageExceptions
await BlobHelpers.GetStorageRetryPolicy($"BlobBlockRange::GetBlockRangeData for source blob {this.sourceBlob.Name} corresponding to block Id {this.Name}", retryCount, logger).ExecuteAsync(async() =>
{
// we do not wrap this around in a 'using' block because the caller will be calling Dispose() on the memory stream
var memStream = new MemoryStream((int)this.Length);
logger.LogDebug($"Inside BlobBlockRange::GetBlockRangeData; about to call DownloadRangeToStreamAsync for {this.Name}");
var blobReqOpts = new BlobRequestOptions()
{
ServerTimeout = TimeSpan.FromSeconds(timeoutSeconds),
MaximumExecutionTime = TimeSpan.FromSeconds(timeoutSeconds)
};
if (!useInbuiltRetry)
{
blobReqOpts.RetryPolicy = new WindowsAzure.Storage.RetryPolicies.NoRetry();
}
await this.sourceBlob.DownloadRangeToStreamAsync(memStream,
this.StartOffset,
this.Length,
null,
blobReqOpts,
null);
logger.LogDebug($"BlobBlockRange::GetBlockRangeData finished DownloadRangeToStreamAsync for {this.Name}");
var encodedChecksum = calcMD5ForBlock ? this.ComputeChecksumFromStream(memStream) : null;
// reset the stream position back to 0
memStream.Position = 0;
retVal = new BlockRangeData()
{
MemStream = memStream,
Base64EncodedMD5Checksum = encodedChecksum
};
});
return(retVal);
}
internal static async Task <bool> ProcessSourceBlocks(Dictionary <string, List <BlockRangeBase> > sourceBlockList,
CloudBlockBlob destBlob,
List <string> destBlockList,
List <string> finalBlockList,
bool calcMD5ForBlock,
int timeoutSeconds,
int maxDOP,
bool useInbuiltRetry,
int retryCount,
OpProgress opProgress,
IProgress <OpProgress> progress,
ILogger logger)
{
foreach (var currBlobItem in sourceBlockList.Keys)
{
var blockRanges = new List <BlockRangeBase>();
var sourceBlobName = currBlobItem;
opProgress.StatusMessage = $"Working on {sourceBlobName}";
logger.LogDebug($"START: {sourceBlobName}");
if (sourceBlockList[currBlobItem].Count == 0)
{
logger.LogError($"There are no block ranges for source item {sourceBlobName}; exiting!");
return(false);
}
foreach (var newBlockRange in sourceBlockList[currBlobItem])
{
// check if this block has already been copied + committed at the destination, and in that case, skip it
if (destBlockList.Contains(newBlockRange.Name))
{
logger.LogDebug($"Destination already has blockID {newBlockRange.Name} for source item {sourceBlobName}; skipping");
continue;
}
else
{
logger.LogDebug($"Adding blockID {newBlockRange.Name} for source item {sourceBlobName} to work list");
blockRanges.Add(newBlockRange);
}
}
logger.LogDebug($"Total number of of ranges to process for source item {sourceBlobName} is {blockRanges.Count}");
// add this list of block IDs to the final list
finalBlockList.AddRange(blockRanges.Select(e => e.Name));
if (blockRanges.Count() > 0)
{
// call the helper function to actually execute the writes to blob
var processBRStatus = await ProcessBlockRanges(blockRanges,
destBlob,
calcMD5ForBlock,
logger,
progress,
opProgress,
timeoutSeconds,
maxDOP,
useInbuiltRetry,
retryCount);
if (!processBRStatus)
{
logger.LogError($"One or more errors encountered when calling ProcessBlockRanges for source item {sourceBlobName}; exiting!");
return(false);
}
// each iteration (each source item) we will commit an ever-increasing super-set of block IDs
// we do this to support "resume" operations later on.
// we will only do this if we actually did any work here TODO review
await BlobHelpers.GetStorageRetryPolicy($"PutBlockListAsync for blob {currBlobItem}", retryCount, logger).ExecuteAsync(async() =>
{
await destBlob.PutBlockListAsync(finalBlockList);
});
}
else
{
logger.LogDebug($"There was no work to be done for source item {currBlobItem} as all blocks already existed in destination.");
}
logger.LogDebug($"END: {currBlobItem}");
// report progress to caller
opProgress.StatusMessage = $"Finished with {currBlobItem}";
progress.Report(opProgress);
}
return(true);
}