/// <summary>
        /// Performs a parallel upload operation on a block blob using the associated serviceclient configuration
        /// </summary>
        /// <param name="blobRef">The reference to the blob.</param>
        /// <param name="sourceStream">The source data to upload.</param>
        /// <param name="options">BlobRequestOptions to use for each upload, can be null.</param>
        /// <summary>
        /// Performs a parallel upload operation on a block blob using the associated serviceclient configuration
        /// </summary>
        /// <param name="blobRef">The reference to the blob.</param>
        /// <param name="sourceStream">The source data to upload.</param>
        /// <param name="blockIdSequenceNumber">The intial block ID, each subsequent block will increment of this value </param>
        /// <param name="options">BlobRequestOptions to use for each upload, can be null.</param>
        public static void ParallelUpload(this CloudBlockBlob blobRef, Stream sourceStream, long blockIdSequenceNumber, BlobRequestOptions options)
        {
            // Parameter Validation & Locals
            if (null == blobRef.ServiceClient)
            {
                throw new ArgumentException("Blob Reference must have a valid service client associated with it");
            }

            if (sourceStream.Length - sourceStream.Position == 0)
            {
                throw new ArgumentException("Cannot upload empty stream.");
            }

            if (null == options)
            {
                options = new BlobRequestOptions()
                {
                    Timeout = blobRef.ServiceClient.Timeout,
                    RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff)
                };
            }

            bool moreToUpload = true;
            List<IAsyncResult> asyncResults = new List<IAsyncResult>();
            List<string> blockList = new List<string>();

            using (MD5 fullBlobMD5 = MD5.Create())
            {
                do
                {
                    int currentPendingTasks = asyncResults.Count;

                    for (int i = currentPendingTasks; i < blobRef.ServiceClient.ParallelOperationThreadCount && moreToUpload; i++)
                    {
                        // Step 1: Create block streams in a serial order as stream can only be read sequentially
                        string blockId = null;

                        // Dispense Block Stream
                        int blockSize = (int)blobRef.ServiceClient.WriteBlockSizeInBytes;
                        int totalCopied = 0, numRead = 0;
                        MemoryStream blockAsStream = null;
                        blockIdSequenceNumber++;

                        int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position);
                        byte[] buffer = new byte[blockBufferSize];
                        blockAsStream = new MemoryStream(buffer);

                        do
                        {
                            numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied);
                            totalCopied += numRead;
                        }
                        while (numRead != 0 && totalCopied < blockBufferSize);

                        // Update Running MD5 Hashes
                        fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0);
                        blockId = GenerateBase64BlockID(blockIdSequenceNumber);

                        // Step 2: Fire off consumer tasks that may finish on other threads
                        blockList.Add(blockId);
                        IAsyncResult asyncresult = blobRef.BeginPutBlock(blockId, blockAsStream, null, options, null, blockAsStream);
                        asyncResults.Add(asyncresult);

                        if (sourceStream.Length == sourceStream.Position)
                        {
                            // No more upload tasks
                            moreToUpload = false;
                        }
                    }

                    // Step 3: Wait for 1 or more put blocks to finish and finish operations
                    if (asyncResults.Count > 0)
                    {
                        int waitTimeout = options.Timeout.HasValue ? (int)Math.Ceiling(options.Timeout.Value.TotalMilliseconds) : Timeout.Infinite;
                        int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);

                        if (waitResult == WaitHandle.WaitTimeout)
                        {
                            throw new TimeoutException(String.Format("ParallelUpload Failed with timeout = {0}", options.Timeout.Value));
                        }

                        // Optimize away any other completed operations
                        for (int index = 0; index < asyncResults.Count; index++)
                        {
                            IAsyncResult result = asyncResults[index];
                            if (result.IsCompleted)
                            {
                                // Dispose of memory stream
                                (result.AsyncState as IDisposable).Dispose();
                                asyncResults.RemoveAt(index);
                                blobRef.EndPutBlock(result);
                                index--;
                            }
                        }
                    }
                }
                while (moreToUpload || asyncResults.Count != 0);

                // Step 4: Calculate MD5 and do a PutBlockList to commit the blob
                fullBlobMD5.TransformFinalBlock(new byte[0], 0, 0);
                byte[] blobHashBytes = fullBlobMD5.Hash;
                string blobHash = Convert.ToBase64String(blobHashBytes);
                blobRef.Properties.ContentMD5 = blobHash;
                blobRef.PutBlockList(blockList, options);
            }
        }
コード例 #2
0
        /// <summary>
        ///     Uploads a single block asynchronously.
        /// </summary>
        /// <param name="blockBlob">Cloud block blob.</param>
        /// <param name="blockId">A base64-encoded block ID that identifies the block.</param>
        /// <param name="blockData">A stream that provides the data for the block.</param>
        /// <param name="contentMd5">
        ///     An optional hash value that will be used to set the
        ///     <see
        ///         cref="P:Microsoft.WindowsAzure.Storage.Blob.BlobProperties.ContentMD5" />
        ///     property
        ///     on the blob. May be <c>null</c> or an empty string.
        /// </param>
        /// <param name="accessCondition">
        ///     An <see cref="T:Microsoft.WindowsAzure.Storage.AccessCondition" /> object that represents the access conditions for the blob. If <c>null</c>, no condition is used.
        /// </param>
        /// <param name="cancellationToken">Cancellation token.</param>
        public static Task PutBlockAsync(
            this CloudBlockBlob blockBlob,
            string blockId,
            Stream blockData,
            string contentMd5,
            AccessCondition accessCondition = null,
            CancellationToken cancellationToken = default (CancellationToken))
        {
            ICancellableAsyncResult asyncResult = blockBlob.BeginPutBlock(blockId, blockData, contentMd5, accessCondition, null, null, null, null);
            CancellationTokenRegistration registration = cancellationToken.Register(p => asyncResult.Cancel(), null);

            return Task.Factory.FromAsync(
                asyncResult,
                result =>
                    {
                        registration.Dispose();
                        blockBlob.EndPutBlock(result);
                    });
        }
コード例 #3
0
        private static void ParallelUpload(this CloudBlockBlob blobRef, Stream sourceStream, UploadInfo uploadInfo, BlobRequestOptions options)
        {
            List<IAsyncResult> asyncResults = new List<IAsyncResult>();
            List<BlockInfo> blockInfoList = uploadInfo.BlockInfoList;

            // set stream position based on read uploadInfo
            int blockSize = (int)blobRef.ServiceClient.WriteBlockSizeInBytes;
            bool moreToUpload = (sourceStream.Length - sourceStream.Position > 0);
            int currentBlockPossition = blockInfoList.Count;

            long totalBytes = sourceStream.Length;
            long uploadedBytes = 0;

            using (MD5 fullBlobMD5 = MD5.Create())
            {
                // re-create file hash if starting again
                if (currentBlockPossition > 0)
                {
                    for (int i = 0; i < currentBlockPossition; i++)
                    {
                        int totalCopied = 0, numRead = 0;
                        int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position);
                        byte[] buffer = new byte[blockBufferSize];

                        do
                        {
                            numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied);
                            totalCopied += numRead;
                        }
                        while (numRead != 0 && totalCopied < blockBufferSize);

                        fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0);
                    }

                    uploadedBytes = sourceStream.Position;
                }

                do
                {
                    int currentPendingTasks = asyncResults.Count;

                    for (int i = currentPendingTasks; i < blobRef.ServiceClient.ParallelOperationThreadCount && moreToUpload; i++)
                    {
                        // Step 1: Create block streams in a serial order as stream can only be read sequentially
                        string blockId = null;

                        // Dispense Block Stream
                        int totalCopied = 0, numRead = 0;
                        MemoryStream blockAsStream = null;
                        uploadInfo.BlockIdSequenceNumber++;

                        int blockBufferSize = (int)Math.Min(blockSize, sourceStream.Length - sourceStream.Position);
                        byte[] buffer = new byte[blockBufferSize];
                        blockAsStream = new MemoryStream(buffer);

                        do
                        {
                            numRead = sourceStream.Read(buffer, totalCopied, blockBufferSize - totalCopied);
                            totalCopied += numRead;
                        }
                        while (numRead != 0 && totalCopied < blockBufferSize);

                        // Update Running MD5 Hashes
                        fullBlobMD5.TransformBlock(buffer, 0, totalCopied, null, 0);
                        blockId = GenerateBase64BlockID(uploadInfo.BlockIdSequenceNumber);

                        // Step 2: Fire off consumer tasks that may finish on other threads
                        BlockInfo blockInfo = new BlockInfo { OrderPosition = currentBlockPossition++, BlockId = blockId };
                        blockInfoList.Add(blockInfo);

                        IAsyncResult asyncresult = blobRef.BeginPutBlock(blockId, blockAsStream, null, options, null,
                            new UploadState { BlockAsStream = blockAsStream, BlockInfo = blockInfo });
                        asyncResults.Add(asyncresult);

                        if (sourceStream.Length == sourceStream.Position)
                        {
                            // No more upload tasks
                            moreToUpload = false;
                        }
                    }

                    // Step 3: Wait for 1 or more put blocks to finish and finish operations
                    if (asyncResults.Count > 0)
                    {
                        int waitTimeout = options.Timeout.HasValue ? (int)Math.Ceiling(options.Timeout.Value.TotalMilliseconds) : Timeout.Infinite;
                        int waitResult = WaitHandle.WaitAny(asyncResults.Select(result => result.AsyncWaitHandle).ToArray(), waitTimeout);

                        if (waitResult == WaitHandle.WaitTimeout)
                        {
                            throw new TimeoutException(String.Format("ParallelUpload Failed with timeout = {0}", options.Timeout.Value));
                        }

                        // Optimize away any other completed operations
                        for (int index = 0; index < asyncResults.Count; index++)
                        {
                            IAsyncResult result = asyncResults[index];
                            if (result.IsCompleted)
                            {
                                // Dispose of memory stream
                                var uploadState = result.AsyncState as UploadState;
                                uploadedBytes += uploadState.BlockAsStream.Length;
                                (uploadState.BlockAsStream as IDisposable).Dispose();

                                asyncResults.RemoveAt(index);
                                blobRef.EndPutBlock(result);
                                index--;

                                // log uploaded block
                                UploadInfo.LogUploadProgress(uploadInfo.LogFilename, uploadState.BlockInfo);

                                // output progress
                                Console.Write("\b\b\b\b");
                                Console.Write(" {0}%", (uploadedBytes * 100) / (totalBytes));
                            }
                        }
                    }
                }
                while (moreToUpload || asyncResults.Count != 0);

                // Step 4: Calculate MD5 and do a PutBlockList to commit the blob
                fullBlobMD5.TransformFinalBlock(new byte[0], 0, 0);
                byte[] blobHashBytes = fullBlobMD5.Hash;
                string blobHash = Convert.ToBase64String(blobHashBytes);
                blobRef.Properties.ContentMD5 = blobHash;

                List<string> blockList = blockInfoList.OrderBy(b => b.OrderPosition).Select(b => b.BlockId).ToList();
                blobRef.PutBlockList(blockList, options);
            }
        }