public async Task <ExecutionResult> UploadToAsync(FileInfo file, Uri blobAbsoluteUri, string prefix = "")
{
try
{
var blockBlob = new CloudBlockBlob(blobAbsoluteUri);
await LargeFileUploaderUtils.UploadAsync(file : file, blockBlob : blockBlob, uploadParallelism : 4);
return(new ExecutionResult {
Success = true, Output = $"{prefix}: Uploaded {file.FullName} to {blobAbsoluteUri.AbsoluteUri}"
});
}
catch (Exception ex)
{
return(new ExecutionResult {
Success = false, Output = $"{prefix}: ERR during upload: \"{ex.Message}\" {blobAbsoluteUri.AbsoluteUri}"
});
}
}
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth,
CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM)
{
const int MAXIMUM_UPLOAD_SIZE = 4 * MB;
if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE)
{
NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE;
}
#region Which blocks exist in the file
var allBlockInFile = Enumerable
.Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk)))
.Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk))
.Where(block => block.Length > 0)
.ToList();
var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList();
#endregion
#region Which blocks are already uploaded
List <BlockMetadata> missingBlocks = null;
try
{
var existingBlocks = (await blockBlob.DownloadBlockListAsync(
BlockListingFilter.Uncommitted,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions {
},
new OperationContext {
}))
.Where(_ => _.Length == NumBytesPerChunk)
.ToList();
missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock =>
existingBlock.Name == blockInFile.BlockId &&
existingBlock.Length == blockInFile.Length)).ToList();
}
catch (StorageException)
{
missingBlocks = allBlockInFile;
}
#endregion
Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) =>
{
byte[] blockData = await fetchLocalData(block.Index, block.Length);
string contentHash = md5()(blockData);
DateTime start = DateTime.UtcNow;
await ExecuteUntilSuccessAsync(async() =>
{
await blockBlob.PutBlockAsync(
blockId: block.BlockId,
blockData: new MemoryStream(blockData, true),
contentMD5: contentHash,
accessCondition: AccessCondition.GenerateEmptyCondition(),
options: new BlobRequestOptions
{
StoreBlobContentMD5 = true,
UseTransactionalMD5 = true
},
operationContext: new OperationContext());
}, consoleExceptionHandler);
stats.Add(block.Length, start);
};
var s = new Statistics(missingBlocks.Sum(b => (long)b.Length));
await LargeFileUploaderUtils.ForEachAsync(
source : missingBlocks,
parallelUploads : 4,
body : blockMetadata => uploadBlockAsync(blockMetadata, s));
await ExecuteUntilSuccessAsync(async() =>
{
await blockBlob.PutBlockListAsync(blockIdList);
}, consoleExceptionHandler);
log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri);
return(blockBlob.Uri.AbsoluteUri);
}
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth,
CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM)
{
const int MAXIMUM_UPLOAD_SIZE = 100 * MB;
if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE)
{
NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE;
}
#region Which blocks exist in the file
var allBlockInFile = Enumerable
.Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk)))
.Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk))
.Where(block => block.Length > 0)
.ToList();
var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList();
#endregion
#region Which blocks are already uploaded
List <BlockMetadata> missingBlocks = null;
try
{
var existingBlocks = (await blockBlob.DownloadBlockListAsync(
BlockListingFilter.Uncommitted,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions {
},
new OperationContext {
}))
.Where(_ => _.Length == NumBytesPerChunk)
.ToList();
missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock =>
existingBlock.Name == blockInFile.BlockId &&
existingBlock.Length == blockInFile.Length)).ToList();
}
catch (StorageException)
{
missingBlocks = allBlockInFile;
}
#endregion
Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) =>
{
DateTime start = DateTime.UtcNow;
stats.Add(block.Length, start);
var fetchLocalDataResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync <byte[]>(() => fetchLocalData(block.Index, block.Length));
if (fetchLocalDataResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not read local file", fetchLocalDataResult.FinalException);
}
byte[] blockData = fetchLocalDataResult.Result;
string contentHash = md5()(blockData);
var putBlockAsyncResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync(() => blockBlob.PutBlockAsync(
blockId: block.BlockId,
blockData: new MemoryStream(blockData, true),
contentMD5: contentHash,
accessCondition: AccessCondition.GenerateEmptyCondition(),
options: new BlobRequestOptions {
StoreBlobContentMD5 = true, UseTransactionalMD5 = true
},
operationContext: new OperationContext()));
if (putBlockAsyncResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not call PutBlockAsync {putBlockAsyncResult.FinalException.Message}", putBlockAsyncResult.FinalException);
}
};
var s = new Statistics(missingBlocks.Sum(b => (long)b.Length));
await LargeFileUploaderUtils.ForEachAsync(
source : missingBlocks,
parallelTasks : 4,
task : blockMetadata => uploadBlockAsync(blockMetadata, s));
var putBlockListAsyncResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync(() => blockBlob.PutBlockListAsync(blockIdList));
if (putBlockListAsyncResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not call PutBlockListAsync", putBlockListAsyncResult.FinalException);
}
log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri);
return(blockBlob.Uri.AbsoluteUri);
}
