internal static async Task <(List <string> blockList, CloudBlockBlob destBlob)> GetDestinationBlobBlockList(string destStorageAccountName,
string destStorageContainerName,
string destBlobName,
string destSAS,
string destStorageAccountKey,
string destEndpointSuffix,
bool overwriteDest,
int retryCount,
ILogger logger)
{
var destBlockList = new List <string>();
// get a reference to the destination blob
var destBlob = BlobHelpers.GetBlockBlob(destStorageAccountName,
destStorageContainerName,
destBlobName,
destSAS,
destStorageAccountKey,
destEndpointSuffix,
retryCount,
logger);
if (destBlob is null)
{
logger.LogError($"Failed to get a reference to destination conatiner / blob {destBlobName}; exiting!");
return(destBlockList, destBlob);
}
// check if the blob exists, in which case we need to also get the list of blocks associated with that blob
// this will help to skip blocks which were already completed, and thereby help with resume
var blockList = await BlobHelpers.GetBlockListForBlob(destBlob, retryCount, logger);
if (blockList is null)
{
// this is when the destination blob does not yet exist
logger.LogDebug($"Destination blob {destBlobName} does not exist (block listing returned null).");
}
else
{
// support overwrite by deleting the destination blob
if (overwriteDest)
{
logger.LogDebug($"Destination blob {destBlobName} exists but needs to be deleted as overwrite == true.");
await destBlob.DeleteAsync();
logger.LogDebug($"Destination blob {destBlobName} deleted to prepare for overwrite.");
}
else
{
logger.LogDebug($"Destination blob {destBlobName} exists; trying to get block listing.");
destBlockList = new List <string>(blockList.Select(b => b.Name));
logger.LogDebug($"Destination blob {destBlobName} has {destBlockList.Count} blocks.");
}
}
return(destBlockList, destBlob);
}
/// <summary>
///
/// </summary>
/// <param name="sourceFileName"></param>
/// <param name="currRange"></param>
/// <param name="destBlob"></param>
/// <param name="logger"></param>
/// <returns></returns>
private async static Task <bool> ProcessBlockRange(
BlockRangeBase currRange,
CloudBlockBlob destBlob,
int timeoutSeconds,
bool useInbuiltRetry,
int retryCount,
bool calcMD5ForBlock,
ILogger logger)
{
logger.LogDebug($"Started ProcessBlockRange for {currRange.Name}");
using (var brData = await currRange.GetBlockRangeData(calcMD5ForBlock,
timeoutSeconds,
useInbuiltRetry,
retryCount,
logger))
{
var brDataIsNull = (brData is null) ? "null" : "valid";
logger.LogDebug($"GetBlockRangeData was called for block {currRange.Name} and returned brData {brDataIsNull}");
if (brData is null)
{
logger.LogDebug($"Returning false from GetBlockRangeData for block {currRange.Name} as brData was {brDataIsNull}");
return(false);
}
logger.LogDebug($"Inside ProcessBlockRange, about to start the PutBlockAsync action for {currRange.Name}");
// use retry policy which will automatically handle the throttling related StorageExceptions
await BlobHelpers.GetStorageRetryPolicy($"PutBlockAsync for block {currRange.Name}", retryCount, logger).ExecuteAsync(async() =>
{
logger.LogDebug($"Before PutBlockAsync for {currRange.Name}");
var blobReqOpts = new BlobRequestOptions()
{
MaximumExecutionTime = TimeSpan.FromSeconds(timeoutSeconds)
};
if (!useInbuiltRetry)
{
blobReqOpts.RetryPolicy = new WindowsAzure.Storage.RetryPolicies.NoRetry();
}
// reset the memory stream again to 0
brData.MemStream.Position = 0;
// and then call Azure Storage to put this as a block with the given block ID and MD5 hash
await destBlob.PutBlockAsync(currRange.Name, brData.MemStream, brData.Base64EncodedMD5Checksum,
null, blobReqOpts, null);
logger.LogDebug($"Finished PutBlockAsync for {currRange.Name}");
});
logger.LogDebug($"Finished ProcessBlockRange for {currRange.Name}");
}
return(true);
}
internal async override Task <BlockRangeData> GetBlockRangeData(bool calcMD5ForBlock, int timeoutSeconds, bool useInbuiltRetry, int retryCount, ILogger logger)
{
BlockRangeData retVal = null;
// use retry policy which will automatically handle the throttling related StorageExceptions
await BlobHelpers.GetStorageRetryPolicy($"BlobBlockRange::GetBlockRangeData for source blob {this.sourceBlob.Name} corresponding to block Id {this.Name}", retryCount, logger).ExecuteAsync(async() =>
{
// we do not wrap this around in a 'using' block because the caller will be calling Dispose() on the memory stream
var memStream = new MemoryStream((int)this.Length);
logger.LogDebug($"Inside BlobBlockRange::GetBlockRangeData; about to call DownloadRangeToStreamAsync for {this.Name}");
var blobReqOpts = new BlobRequestOptions()
{
ServerTimeout = TimeSpan.FromSeconds(timeoutSeconds),
MaximumExecutionTime = TimeSpan.FromSeconds(timeoutSeconds)
};
if (!useInbuiltRetry)
{
blobReqOpts.RetryPolicy = new WindowsAzure.Storage.RetryPolicies.NoRetry();
}
await this.sourceBlob.DownloadRangeToStreamAsync(memStream,
this.StartOffset,
this.Length,
null,
blobReqOpts,
null);
logger.LogDebug($"BlobBlockRange::GetBlockRangeData finished DownloadRangeToStreamAsync for {this.Name}");
var encodedChecksum = calcMD5ForBlock ? this.ComputeChecksumFromStream(memStream) : null;
// reset the stream position back to 0
memStream.Position = 0;
retVal = new BlockRangeData()
{
MemStream = memStream,
Base64EncodedMD5Checksum = encodedChecksum
};
});
return(retVal);
}
/// <summary>
/// Concatenate a set of blobs - specified either via their prefix or an explicit list of blob names - to a single destination blob
/// Source blobs must be from a single container. However because of the way the Azure Storage Block Blob works, you can call
/// this function multiple times, specifying different sets of source blobs each time, and they will simply be "appended" to the destination blob.
/// </summary>
/// <param name="sourceStorageAccountName"></param>
/// <param name="sourceStorageContainerName"></param>
/// <param name="sourceStorageAccountKey"></param>
/// <param name="isSourceSAS"></param>
/// <param name="sourceBlobPrefix"></param>
/// <param name="sourceEndpointSuffix"></param>
/// <param name="sortBlobs"></param>
/// <param name="sourceSAS"></param>
/// <param name="sourceBlobNames"></param>
/// <param name="destEndpointSuffix"></param>
/// <param name="destSAS"></param>
/// <param name="destStorageAccountName"></param>
/// <param name="destStorageAccountKey"></param>
/// <param name="isDestSAS"></param>
/// <param name="destStorageContainerName"></param>
/// <param name="destBlobName"></param>
/// <param name="colHeader"></param>
/// <param name="calcMD5ForBlocks"></param>
/// <param name="logger"></param>
/// <returns></returns>
public async static Task <bool> BlobToBlob(string sourceStorageAccountName,
string sourceStorageContainerName,
string sourceStorageAccountKey,
string sourceSAS,
string sourceBlobPrefix,
string sourceEndpointSuffix,
bool sortBlobs,
List <string> sourceBlobNames,
string destStorageAccountName,
string destStorageAccountKey,
string destSAS,
string destStorageContainerName,
string destBlobName,
string destEndpointSuffix,
string colHeader,
string fileSeparator,
bool calcMD5ForBlock,
bool overwriteDest,
int timeoutSeconds,
int maxDOP,
bool useInbuiltRetry,
int retryCount,
ILogger logger,
IProgress <OpProgress> progress)
{
var opProgress = new OpProgress();
try
{
var sw = Stopwatch.StartNew();
GlobalOptimizations();
var res = await BlobHelpers.GetDestinationBlobBlockList(destStorageAccountName,
destStorageContainerName,
destBlobName,
destSAS,
destStorageAccountKey,
destEndpointSuffix,
overwriteDest,
retryCount,
logger);
var destBlockList = res.blockList;
var destBlob = res.destBlob;
if (destBlockList is null)
{
return(false);
}
// create a place holder for the final block list (to be eventually used for put block list) and pre-populate it with the known list of blocks
// already associated with the destination blob
var finalBlockList = new List <string>(destBlockList);
// signal back to the caller with the total block list
// check if there is a specific list of blobs given by the user, in which case the immediate 'if' code below will be skipped
if (sourceBlobNames is null || sourceBlobNames.Count == 0)
{
// now just get the blob names, that's all we need for further processing
sourceBlobNames = await BlobHelpers.GetBlobListing(sourceStorageAccountName,
sourceStorageContainerName,
sourceBlobPrefix,
sourceSAS,
sourceStorageAccountKey,
sourceEndpointSuffix,
retryCount,
logger);
// check for null being returned from above in which case we need to exit
if (sourceBlobNames is null)
{
logger.LogError($"Souce blob listing failed to return any results. Exiting!");
return(false);
}
// if the user specified to sort the input blobs (only valid for the prefix case) then we will happily do that!
// The gotcha here is that this is a string sort. So if blobs have names like blob_9, blob_13, blob_6, blob_3, blob_1
// the sort order will result in blob_1, blob_13, blob_3, blob_6, blob_9.
// To avoid issues like this the user must 0-prefix the numbers embedded in the filenames.
if (sortBlobs)
{
sourceBlobNames.Sort();
}
}
// var sourceBlobItems = new List<BlobItem>();
var sourceBlockList = new Dictionary <string, List <BlockRangeBase> >();
// we first need to seed with the column header if it was specified
PrefixColumnHeaderIfApplicable(sourceBlockList,
colHeader,
sourceEndpointSuffix,
sourceStorageAccountName,
sourceStorageContainerName);
// it is necessary to use a dictionary to hold the BlobItem because we do want to preserve sort order if necessary
int tmpBlobIndex = 0;
foreach (var srcBlob in sourceBlobNames)
{
sourceBlockList.Add(srcBlob, new List <BlockRangeBase>());
InjectFileSeparator(sourceBlockList,
fileSeparator,
tmpBlobIndex,
sourceEndpointSuffix,
sourceStorageAccountName,
sourceStorageContainerName);
tmpBlobIndex++;
}
// sourceBlobItems.AddRange(sourceBlobNames.Select(b => new BlobItem { sourceBlobName = b }));
// get block lists for all source blobs in parallel. earlier this was done serially and was found to be bottleneck
tmpBlobIndex = 0;
Parallel.ForEach(sourceBlobNames, srcBlobName =>
{
Interlocked.Increment(ref tmpBlobIndex);
var tmpSrcBlob = BlobHelpers.GetBlockBlob(sourceStorageAccountName,
sourceStorageContainerName,
srcBlobName,
sourceSAS,
sourceStorageAccountKey,
sourceEndpointSuffix,
retryCount,
logger);
if (tmpSrcBlob is null)
{
// throw exception. this condition will only be entered if a blob name was explicitly specified and it does not really exist
throw new StorageException($"An invalid source blob ({srcBlobName}) was specified.");
}
// first we get the block list of the source blob. we use this to later parallelize the download / copy operation
var tmpBlockList = BlobHelpers.GetBlockListForBlob(tmpSrcBlob, retryCount, logger).GetAwaiter().GetResult();
// proceed to construct a List<BlobBlockRange> to add to the master dictionary
// iterate through the list of blocks and compute their effective offsets in the final file.
long currentOffset = 0;
int chunkIndex = 0;
var blocksToBeAdded = new List <BlobBlockRange>();
if (tmpBlockList.Count() == 0 && tmpSrcBlob.Properties.Length > 0)
{
// in case the source blob is smaller then 256 MB (for latest API) then the blob is stored directly without any block list
// so in this case the sourceBlockList is 0-length and we need to fake a BlockListItem as null, which will later be handled below
var blockLength = tmpSrcBlob.Properties.Length;
blocksToBeAdded.Add(new BlobBlockRange(tmpSrcBlob,
string.Concat(
tmpBlobIndex,
sourceEndpointSuffix,
sourceStorageAccountName,
sourceStorageContainerName,
srcBlobName,
blockLength,
chunkIndex),
currentOffset,
blockLength));
}
else
{
foreach (var blockListItem in tmpBlockList)
{
var blockLength = blockListItem.Length;
// compute a unique blockId based on blob account + container + blob name (includes path) + block length + block "number"
// We also add a fileIndex component (tmpBlockIndex), to allow for the same source blob to recur in the list of source blobs
blocksToBeAdded.Add(new BlobBlockRange(tmpSrcBlob,
string.Concat(
tmpBlobIndex,
sourceEndpointSuffix,
sourceStorageAccountName,
sourceStorageContainerName,
srcBlobName,
blockLength,
chunkIndex),
currentOffset,
blockLength));
// increment this here itself as we may potentially skip to the next blob
chunkIndex++;
currentOffset += blockLength;
}
}
lock (sourceBlockList)
{
sourceBlockList[srcBlobName].AddRange(blocksToBeAdded);
}
});
// the total number of "ticks" to be reported will be the number of blocks + the number of blobs
// this is because each PutBlock is reported separately, as is the PutBlockList when each source blob is finished
opProgress.TotalTicks = sourceBlockList.Count + sourceBlockList.Values.Select(b => b.Count()).Sum();
progress.Report(opProgress);
if (!await BlockRangeWorkers.ProcessSourceBlocks(sourceBlockList,
destBlob,
destBlockList,
finalBlockList,
calcMD5ForBlock,
timeoutSeconds,
maxDOP,
useInbuiltRetry,
retryCount,
opProgress,
progress,
logger))
{
return(false);
}
sw.Stop();
opProgress.StatusMessage = $"BlobToBlob operation suceeded in {sw.Elapsed.TotalSeconds} seconds.";
progress.Report(opProgress);
}
catch (StorageException ex)
{
//opProgress.Percent = 100;
//opProgress.StatusMessage = "Errors occured. Details in the log.";
//progress.Report(opProgress);
BlobHelpers.LogStorageException("Unhandled exception in BlobToBlob", ex, logger, false);
return(false);
}
return(true);
}
/// <summary>
/// Concatenates a set of files on disk into a single block blob in Azure Storage.
/// The destination block blob can exist; in which case the files will be appended to the existing blob.
/// </summary>
/// <param name="sourceFolderName"></param>
/// <param name="sourceBlobPrefix"></param>
/// <param name="sortFiles"></param>
/// <param name="sourceFileNames"></param>
/// <param name="destStorageAccountName"></param>
/// <param name="destStorageAccountKey"></param>
/// <param name="destSAS"></param>
/// <param name="destStorageContainerName"></param>
/// <param name="destBlobName"></param>
/// <param name="destEndpointSuffix"></param>
/// <param name="colHeader"></param>
/// <param name="calcMD5ForBlocks"></param>
/// <param name="logger"></param>
/// <returns>
/// True if successful; False if errors found
/// </returns>
public static async Task <bool> DiskToBlob(string sourceFolderName,
string sourceBlobPrefix,
bool sortFiles,
List <string> sourceFileNames,
string destStorageAccountName,
string destStorageAccountKey,
string destSAS,
string destStorageContainerName,
string destBlobName,
string destEndpointSuffix,
string colHeader,
string fileSeparator,
bool calcMD5ForBlock,
bool overwriteDest,
int timeoutSeconds,
int maxDOP,
bool useInbuiltRetry,
int retryCount,
ILogger logger,
IProgress <OpProgress> progress
)
{
var opProgress = new OpProgress();
try
{
var sw = Stopwatch.StartNew();
GlobalOptimizations();
var res = await BlobHelpers.GetDestinationBlobBlockList(destStorageAccountName,
destStorageContainerName,
destBlobName,
destSAS,
destStorageAccountKey,
destEndpointSuffix,
overwriteDest,
retryCount,
logger);
var destBlockList = res.blockList;
var destBlob = res.destBlob;
if (destBlockList is null)
{
return(false);
}
// this will start off as blank; we will keep appending to this the list of block IDs for each file
var finalBlockList = new List <string>();
// check if there is a specific list of files given by the user, in which case the immediate 'if' code below will be skipped
if (sourceFileNames is null || sourceFileNames.Count == 0)
{
// we have a prefix specified, so get a of blobs with a specific prefix and then add them to a list
sourceFileNames = Directory.GetFiles(sourceFolderName, sourceBlobPrefix + "*").ToList();
// if the user specified to sort the input blobs (only valid for the prefix case) then we will happily do that!
// The gotcha here is that this is a string sort. So if files have names like file_9, file_13, file_6, file_3, file_1
// the sort order will result in file_1, file_13, file_3, file_6, file_9.
// To avoid issues like this the user must 0-prefix the numbers embedded in the filenames.
if (sortFiles)
{
sourceFileNames.Sort();
}
}
var sourceBlockList = new Dictionary <string, List <BlockRangeBase> >();
// we first need to seed with the column header if it was specified
PrefixColumnHeaderIfApplicable(sourceBlockList,
colHeader,
null,
null,
null);
int tmpFileIndex = 0;
// it is necessary to use a dictionary to hold the BlobItem because we do want to preserve sort order if necessary
foreach (var srcFile in sourceFileNames)
{
sourceBlockList.Add(srcFile, new List <BlockRangeBase>());
InjectFileSeparator(sourceBlockList,
fileSeparator,
tmpFileIndex,
null,
null,
null);
tmpFileIndex++;
}
foreach (var sourceFileName in sourceFileNames)
{
var fInfo = new FileInfo(sourceFileName);
// construct the set of "block ranges" by splitting the source files into chunks
// of CHUNK_SIZE or the remaining length, as appropriate.
var chunkSet = Enumerable.Range(0, (int)(fInfo.Length / CHUNK_SIZE) + 1)
.Select(ci => new FileBlockRange(sourceFileName,
string.Concat(sourceFileName,
fInfo.Length,
ci),
ci * CHUNK_SIZE,
(ci * CHUNK_SIZE + CHUNK_SIZE > fInfo.Length) ?
fInfo.Length - ci * CHUNK_SIZE :
CHUNK_SIZE
));
sourceBlockList[sourceFileName].AddRange(chunkSet);
}
// the total number of "ticks" to be reported will be the number of blocks + the number of blobs
// this is because each PutBlock is reported separately, as is the PutBlockList when each source blob is finished
opProgress.TotalTicks = sourceBlockList.Count + sourceBlockList.Values.Select(b => b.Count()).Sum();
progress.Report(opProgress);
if (!await BlockRangeWorkers.ProcessSourceBlocks(sourceBlockList,
destBlob,
destBlockList,
finalBlockList,
calcMD5ForBlock,
timeoutSeconds,
maxDOP,
useInbuiltRetry,
retryCount,
opProgress,
progress,
logger))
{
return(false);
}
sw.Stop();
opProgress.StatusMessage = $"DiskToBlob operation suceeded in {sw.Elapsed.TotalSeconds} seconds.";
}
catch (StorageException ex)
{
opProgress.Percent = 100;
opProgress.StatusMessage = "Errors occured. Details in the log.";
progress.Report(opProgress);
BlobHelpers.LogStorageException("Unhandled exception in DiskToBlob", ex, logger, false);
return(false);
}
return(true);
}
internal static async Task <bool> ProcessSourceBlocks(Dictionary <string, List <BlockRangeBase> > sourceBlockList,
CloudBlockBlob destBlob,
List <string> destBlockList,
List <string> finalBlockList,
bool calcMD5ForBlock,
int timeoutSeconds,
int maxDOP,
bool useInbuiltRetry,
int retryCount,
OpProgress opProgress,
IProgress <OpProgress> progress,
ILogger logger)
{
foreach (var currBlobItem in sourceBlockList.Keys)
{
var blockRanges = new List <BlockRangeBase>();
var sourceBlobName = currBlobItem;
opProgress.StatusMessage = $"Working on {sourceBlobName}";
logger.LogDebug($"START: {sourceBlobName}");
if (sourceBlockList[currBlobItem].Count == 0)
{
logger.LogError($"There are no block ranges for source item {sourceBlobName}; exiting!");
return(false);
}
foreach (var newBlockRange in sourceBlockList[currBlobItem])
{
// check if this block has already been copied + committed at the destination, and in that case, skip it
if (destBlockList.Contains(newBlockRange.Name))
{
logger.LogDebug($"Destination already has blockID {newBlockRange.Name} for source item {sourceBlobName}; skipping");
continue;
}
else
{
logger.LogDebug($"Adding blockID {newBlockRange.Name} for source item {sourceBlobName} to work list");
blockRanges.Add(newBlockRange);
}
}
logger.LogDebug($"Total number of of ranges to process for source item {sourceBlobName} is {blockRanges.Count}");
// add this list of block IDs to the final list
finalBlockList.AddRange(blockRanges.Select(e => e.Name));
if (blockRanges.Count() > 0)
{
// call the helper function to actually execute the writes to blob
var processBRStatus = await ProcessBlockRanges(blockRanges,
destBlob,
calcMD5ForBlock,
logger,
progress,
opProgress,
timeoutSeconds,
maxDOP,
useInbuiltRetry,
retryCount);
if (!processBRStatus)
{
logger.LogError($"One or more errors encountered when calling ProcessBlockRanges for source item {sourceBlobName}; exiting!");
return(false);
}
// each iteration (each source item) we will commit an ever-increasing super-set of block IDs
// we do this to support "resume" operations later on.
// we will only do this if we actually did any work here TODO review
await BlobHelpers.GetStorageRetryPolicy($"PutBlockListAsync for blob {currBlobItem}", retryCount, logger).ExecuteAsync(async() =>
{
await destBlob.PutBlockListAsync(finalBlockList);
});
}
else
{
logger.LogDebug($"There was no work to be done for source item {currBlobItem} as all blocks already existed in destination.");
}
logger.LogDebug($"END: {currBlobItem}");
// report progress to caller
opProgress.StatusMessage = $"Finished with {currBlobItem}";
progress.Report(opProgress);
}
return(true);
}