/// <summary> /// Performs a parallel upload operation on a block blob using the associated serviceclient configuration /// </summary> /// <param name="blobRef">The reference to the blob.</param> /// <param name="sourceStream">The source data to upload.</param> /// <param name="options">BlobRequestOptions to use for each upload, can be null.</param> /// <summary> /// Performs a parallel upload operation on a block blob using the associated serviceclient configuration /// </summary> /// <param name="blobRef">The reference to the blob.</param> /// <param name="sourceStream">The source data to upload.</param> /// <param name="blockIdSequenceNumber">The intial block ID, each subsequent block will increment of this value </param> /// <param name="options">BlobRequestOptions to use for each upload, can be null.</param> public static void ParallelUpload(this CloudBlockBlob blobRef, Stream sourceStream, long blockIdSequenceNumber, BlobRequestOptions options) { // Parameter Validation & Locals if (null == blobRef.ServiceClient) { throw new ArgumentException("Blob Reference must have a valid service client associated with it"); } if (sourceStream.Length - sourceStream.Position == 0) { throw new ArgumentException("Cannot upload empty stream."); } if (null == options) { options = new BlobRequestOptions() { Timeout = blobRef.ServiceClient.Timeout, RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff) }; } bool moreToUpload = true; List<IAsyncResult> asyncResults = new List<IAsyncResult>(); List<string> blockList = new List<string>(); using (MD5 fullBlobMD5 = MD5.Create()) { do { int currentPendingTasks = asyncResults.Count; for (int i = currentPendingTasks; i < blobRef.ServiceClient.ParallelOperationThreadCount && moreToUpload; i++) { // Step 1: Create block streams in a serial order as stream can only be read sequentially string blockId = null; // Dispense Block Stream int blockSize = (int)blobRef.ServiceClient.WriteBlockSizeInBytes; int totalCopied = 0, numRead = 0; MemoryStream blockAsStream = null; blockIdSequenceNumber++; int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position); byte[] buffer = new byte[blockBufferSize]; blockAsStream = new MemoryStream(buffer); do { numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied); totalCopied += numRead; } while (numRead != 0 && totalCopied < blockBufferSize); // Update Running MD5 Hashes fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0); blockId = GenerateBase64BlockID(blockIdSequenceNumber); // Step 2: Fire off consumer tasks that may finish on other threads blockList.Add(blockId); IAsyncResult asyncresult = blobRef.BeginPutBlock(blockId, blockAsStream, null, options, null, blockAsStream); asyncResults.Add(asyncresult); if (sourceStream.Length == sourceStream.Position) { // No more upload tasks moreToUpload = false; } } // Step 3: Wait for 1 or more put blocks to finish and finish operations if (asyncResults.Count > 0) { int waitTimeout = options.Timeout.HasValue ? (int)Math.Ceiling(options.Timeout.Value.TotalMilliseconds) : Timeout.Infinite; int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout); if (waitResult == WaitHandle.WaitTimeout) { throw new TimeoutException(String.Format("ParallelUpload Failed with timeout = {0}", options.Timeout.Value)); } // Optimize away any other completed operations for (int index = 0; index < asyncResults.Count; index++) { IAsyncResult result = asyncResults[index]; if (result.IsCompleted) { // Dispose of memory stream (result.AsyncState as IDisposable).Dispose(); asyncResults.RemoveAt(index); blobRef.EndPutBlock(result); index--; } } } } while (moreToUpload || asyncResults.Count != 0); // Step 4: Calculate MD5 and do a PutBlockList to commit the blob fullBlobMD5.TransformFinalBlock(new byte[0], 0, 0); byte[] blobHashBytes = fullBlobMD5.Hash; string blobHash = Convert.ToBase64String(blobHashBytes); blobRef.Properties.ContentMD5 = blobHash; blobRef.PutBlockList(blockList, options); } }
/// <summary> /// Uploads a single block asynchronously. /// </summary> /// <param name="blockBlob">Cloud block blob.</param> /// <param name="blockId">A base64-encoded block ID that identifies the block.</param> /// <param name="blockData">A stream that provides the data for the block.</param> /// <param name="contentMd5"> /// An optional hash value that will be used to set the /// <see /// cref="P:Microsoft.WindowsAzure.Storage.Blob.BlobProperties.ContentMD5" /> /// property /// on the blob. May be <c>null</c> or an empty string. /// </param> /// <param name="accessCondition"> /// An <see cref="T:Microsoft.WindowsAzure.Storage.AccessCondition" /> object that represents the access conditions for the blob. If <c>null</c>, no condition is used. /// </param> /// <param name="cancellationToken">Cancellation token.</param> public static Task PutBlockAsync( this CloudBlockBlob blockBlob, string blockId, Stream blockData, string contentMd5, AccessCondition accessCondition = null, CancellationToken cancellationToken = default (CancellationToken)) { ICancellableAsyncResult asyncResult = blockBlob.BeginPutBlock(blockId, blockData, contentMd5, accessCondition, null, null, null, null); CancellationTokenRegistration registration = cancellationToken.Register(p => asyncResult.Cancel(), null); return Task.Factory.FromAsync( asyncResult, result => { registration.Dispose(); blockBlob.EndPutBlock(result); }); }
private static void ParallelUpload(this CloudBlockBlob blobRef, Stream sourceStream, UploadInfo uploadInfo, BlobRequestOptions options) { List<IAsyncResult> asyncResults = new List<IAsyncResult>(); List<BlockInfo> blockInfoList = uploadInfo.BlockInfoList; // set stream position based on read uploadInfo int blockSize = (int)blobRef.ServiceClient.WriteBlockSizeInBytes; bool moreToUpload = (sourceStream.Length - sourceStream.Position > 0); int currentBlockPossition = blockInfoList.Count; long totalBytes = sourceStream.Length; long uploadedBytes = 0; using (MD5 fullBlobMD5 = MD5.Create()) { // re-create file hash if starting again if (currentBlockPossition > 0) { for (int i = 0; i < currentBlockPossition; i++) { int totalCopied = 0, numRead = 0; int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position); byte[] buffer = new byte[blockBufferSize]; do { numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied); totalCopied += numRead; } while (numRead != 0 && totalCopied < blockBufferSize); fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0); } uploadedBytes = sourceStream.Position; } do { int currentPendingTasks = asyncResults.Count; for (int i = currentPendingTasks; i < blobRef.ServiceClient.ParallelOperationThreadCount && moreToUpload; i++) { // Step 1: Create block streams in a serial order as stream can only be read sequentially string blockId = null; // Dispense Block Stream int totalCopied = 0, numRead = 0; MemoryStream blockAsStream = null; uploadInfo.BlockIdSequenceNumber++; int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position); byte[] buffer = new byte[blockBufferSize]; blockAsStream = new MemoryStream(buffer); do { numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied); totalCopied += numRead; } while (numRead != 0 && totalCopied < blockBufferSize); // Update Running MD5 Hashes fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0); blockId = GenerateBase64BlockID(uploadInfo.BlockIdSequenceNumber); // Step 2: Fire off consumer tasks that may finish on other threads BlockInfo blockInfo = new BlockInfo { OrderPosition = currentBlockPossition++, BlockId = blockId }; blockInfoList.Add(blockInfo); IAsyncResult asyncresult = blobRef.BeginPutBlock(blockId, blockAsStream, null, options, null, new UploadState { BlockAsStream = blockAsStream, BlockInfo = blockInfo }); asyncResults.Add(asyncresult); if (sourceStream.Length == sourceStream.Position) { // No more upload tasks moreToUpload = false; } } // Step 3: Wait for 1 or more put blocks to finish and finish operations if (asyncResults.Count > 0) { int waitTimeout = options.Timeout.HasValue ? (int)Math.Ceiling(options.Timeout.Value.TotalMilliseconds) : Timeout.Infinite; int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout); if (waitResult == WaitHandle.WaitTimeout) { throw new TimeoutException(String.Format("ParallelUpload Failed with timeout = {0}", options.Timeout.Value)); } // Optimize away any other completed operations for (int index = 0; index < asyncResults.Count; index++) { IAsyncResult result = asyncResults[index]; if (result.IsCompleted) { // Dispose of memory stream var uploadState = result.AsyncState as UploadState; uploadedBytes += uploadState.BlockAsStream.Length; (uploadState.BlockAsStream as IDisposable).Dispose(); asyncResults.RemoveAt(index); blobRef.EndPutBlock(result); index--; // log uploaded block UploadInfo.LogUploadProgress(uploadInfo.LogFilename, uploadState.BlockInfo); // output progress Console.Write("\b\b\b\b"); Console.Write(" {0}%", (uploadedBytes * 100) / (totalBytes)); } } } } while (moreToUpload || asyncResults.Count != 0); // Step 4: Calculate MD5 and do a PutBlockList to commit the blob fullBlobMD5.TransformFinalBlock(new byte[0], 0, 0); byte[] blobHashBytes = fullBlobMD5.Hash; string blobHash = Convert.ToBase64String(blobHashBytes); blobRef.Properties.ContentMD5 = blobHash; List<string> blockList = blockInfoList.OrderBy(b => b.OrderPosition).Select(b => b.BlockId).ToList(); blobRef.PutBlockList(blockList, options); } }