public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth, CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM) { const int MAXIMUM_UPLOAD_SIZE = 4 * MB; if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE) { NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE; } #region Which blocks exist in the file var allBlockInFile = Enumerable .Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk))) .Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk)) .Where(block => block.Length > 0) .ToList(); var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList(); #endregion #region Which blocks are already uploaded List <BlockMetadata> missingBlocks = null; try { var existingBlocks = (await blockBlob.DownloadBlockListAsync( BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { }, new OperationContext { })) .Where(_ => _.Length == NumBytesPerChunk) .ToList(); missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock => existingBlock.Name == blockInFile.BlockId && existingBlock.Length == blockInFile.Length)).ToList(); } catch (StorageException) { missingBlocks = allBlockInFile; } #endregion Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) => { byte[] blockData = await fetchLocalData(block.Index, block.Length); string contentHash = md5()(blockData); DateTime start = DateTime.UtcNow; await ExecuteUntilSuccessAsync(async() => { await blockBlob.PutBlockAsync( blockId: block.BlockId, blockData: new MemoryStream(blockData, true), contentMD5: contentHash, accessCondition: AccessCondition.GenerateEmptyCondition(), options: new BlobRequestOptions { StoreBlobContentMD5 = true, UseTransactionalMD5 = true }, operationContext: new OperationContext()); }, consoleExceptionHandler); stats.Add(block.Length, start); }; var s = new Statistics(missingBlocks.Sum(b => (long)b.Length)); await LargeFileUploaderUtils.ForEachAsync( source : missingBlocks, parallelUploads : 4, body : blockMetadata => uploadBlockAsync(blockMetadata, s)); await ExecuteUntilSuccessAsync(async() => { await blockBlob.PutBlockListAsync(blockIdList); }, consoleExceptionHandler); log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri); return(blockBlob.Uri.AbsoluteUri); }
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth, CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM) { const int MAXIMUM_UPLOAD_SIZE = 100 * MB; if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE) { NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE; } #region Which blocks exist in the file var allBlockInFile = Enumerable .Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk))) .Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk)) .Where(block => block.Length > 0) .ToList(); var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList(); #endregion #region Which blocks are already uploaded List <BlockMetadata> missingBlocks = null; try { var existingBlocks = (await blockBlob.DownloadBlockListAsync( BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { }, new OperationContext { })) .Where(_ => _.Length == NumBytesPerChunk) .ToList(); missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock => existingBlock.Name == blockInFile.BlockId && existingBlock.Length == blockInFile.Length)).ToList(); } catch (StorageException) { missingBlocks = allBlockInFile; } #endregion Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) => { DateTime start = DateTime.UtcNow; stats.Add(block.Length, start); var fetchLocalDataResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync <byte[]>(() => fetchLocalData(block.Index, block.Length)); if (fetchLocalDataResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not read local file", fetchLocalDataResult.FinalException); } byte[] blockData = fetchLocalDataResult.Result; string contentHash = md5()(blockData); var putBlockAsyncResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync(() => blockBlob.PutBlockAsync( blockId: block.BlockId, blockData: new MemoryStream(blockData, true), contentMD5: contentHash, accessCondition: AccessCondition.GenerateEmptyCondition(), options: new BlobRequestOptions { StoreBlobContentMD5 = true, UseTransactionalMD5 = true }, operationContext: new OperationContext())); if (putBlockAsyncResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not call PutBlockAsync {putBlockAsyncResult.FinalException.Message}", putBlockAsyncResult.FinalException); } }; var s = new Statistics(missingBlocks.Sum(b => (long)b.Length)); await LargeFileUploaderUtils.ForEachAsync( source : missingBlocks, parallelTasks : 4, task : blockMetadata => uploadBlockAsync(blockMetadata, s)); var putBlockListAsyncResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync(() => blockBlob.PutBlockListAsync(blockIdList)); if (putBlockListAsyncResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not call PutBlockListAsync", putBlockListAsyncResult.FinalException); } log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri); return(blockBlob.Uri.AbsoluteUri); }