private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response, KeyValuePair <long, int> blockOffsetAndLength, ref long bytesDownloaded) { using (Stream stream = response.GetResponseStream()) { int offsetInChunk = 0; int remaining = blockOffsetAndLength.Value; while (remaining > 0) { int read = stream.Read(buffer, offsetInChunk, remaining); lock (lockobject) { fs.Position = blockOffsetAndLength.Key + offsetInChunk; if (fileEncryption != null) { lock (fileEncryption) { using ( FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk) ) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk); } } } fs.Write(buffer, offsetInChunk, read); } offsetInChunk += read; remaining -= read; Interlocked.Add(ref bytesDownloaded, read); } } return(bytesDownloaded); }
protected void ApplyEncryptionTransform(FileEncryption fileEncryption, string fileName, long beginFilePosition, byte[] buffer, int bytesToWrite) { if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(fileName, beginFilePosition)) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: 0, inputCount: bytesToWrite, outputBuffer: buffer, outputOffset: 0); } } } }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subFolder, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read)) { } Exception lastException = null; CloudBlockBlob blob = null; // stats from azurescope show 10 to be an optimal number of transfer threads int numThreads = ParallelTransferThreadCount; var file = new FileInfo(localFile); long fileSize = file.Length; int maxBlockSize = GetBlockSize(fileSize); // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where // the 'key' is block id and 'value' is the block length. List <string> blockList; var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList); int exceptionCount = 0; blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy }); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } var options = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; // Launch threads to upload blocks. var tasks = new List <Task>(); long bytesSent = 0; Action action = () => { List <Exception> exceptions = new List <Exception>(); int sasRetry = 0; if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { Thread.Sleep(_forceSharedAccessSignatureRetry); } if (queue.Count > 0) { FileStream fs = null; try { fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); KeyValuePair <int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); try { var buffer = new byte[blockIdAndLength.Value]; var binaryReader = new BinaryReader(fs); // move the file system reader to the proper position fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value); if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } } using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value)) { string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture)))); string blockHash = GetMd5HashFromStream(buffer); if (blob != null) { blob.PutBlock(blockIdString, ms, blockHash, options: options); } } Interlocked.Add(ref bytesSent, blockIdAndLength.Value); var progress = (int)((double)bytesSent / file.Length * 100); var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null); OnTaskProgressChanged(eArgs); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null) { sasRetry++; if (sasRetry > MaxSasSignatureRetry) { throw; } blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); } else { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions); throw lastException; } Thread.Sleep(tm); } queue.Enqueue(blockIdAndLength); } catch (IOException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.", exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions); throw lastException; } // dispose existing file stream if (fs != null) { fs.Close(); } Thread.Sleep(tm); // try to reopen the file stream again fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); queue.Enqueue(blockIdAndLength); } } } finally { if (fs != null) { fs.Close(); } } } }; for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew( action, cancellationToken, TaskCreationOptions.AttachedToParent, TaskScheduler.Current)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) => { if (result.Any(t => t.IsFaulted)) { return; } try { blob.PutBlockList(blockList, options: options); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null) { blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); blob.PutBlockList(blockList, options: options); } else { throw; } } }, TaskContinuationOptions.None).Wait(cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri); }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subFolder, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read)) { } Exception lastException = null; CloudBlockBlob blob = null; // stats from azurescope show 10 to be an optimal number of transfer threads int numThreads = ParallelTransferThreadCount; var file = new FileInfo(localFile); long fileSize = file.Length; int maxBlockSize = GetBlockSize(fileSize); // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where // the 'key' is block id and 'value' is the block length. List<string> blockList; var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList); int exceptionCount = 0; blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType); blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy }); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } var options = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; // Launch threads to upload blocks. var tasks = new List<Task>(); long bytesSent = 0; Action action = () => { List<Exception> exceptions = new List<Exception>(); if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { Thread.Sleep(_forceSharedAccessSignatureRetry); } if (queue.Count > 0) { FileStream fs = null; try { fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); KeyValuePair<int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); try { var buffer = new byte[blockIdAndLength.Value]; var binaryReader = new BinaryReader(fs); // move the file system reader to the proper position fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value); if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } } using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value)) { string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture)))); string blockHash = GetMd5HashFromStream(buffer); if (blob != null) blob.PutBlock(blockIdString, ms, blockHash, options: options); } Interlocked.Add(ref bytesSent, blockIdAndLength.Value); var progress = (int)((double)bytesSent / file.Length * 100); var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null); OnTaskProgressChanged(eArgs); } catch (StorageException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions); throw lastException; } Thread.Sleep(tm); queue.Enqueue(blockIdAndLength); } catch (IOException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.", exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions); throw lastException; } // dispose existing file stream if (fs != null) { fs.Close(); } Thread.Sleep(tm); // try to reopen the file stream again fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); queue.Enqueue(blockIdAndLength); } } } finally { if (fs != null) { fs.Close(); } } } }; for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew( action, cancellationToken, TaskCreationOptions.AttachedToParent, TaskScheduler.Current)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) => { if (result.Any(t => t.IsFaulted)) { return; } blob.PutBlockList(blockList, options: options); }, TaskContinuationOptions.None).Wait(cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri); }
private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response, KeyValuePair<long, int> blockOffsetAndLength, ref long bytesDownloaded) { using (Stream stream = response.GetResponseStream()) { int offsetInChunk = 0; int remaining = blockOffsetAndLength.Value; while (remaining > 0) { int read = stream.Read(buffer, offsetInChunk, remaining); lock (lockobject) { fs.Position = blockOffsetAndLength.Key + offsetInChunk; if (fileEncryption != null) { lock (fileEncryption) { using ( FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk) ) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk); } } } fs.Write(buffer, offsetInChunk, read); } offsetInChunk += read; remaining -= read; Interlocked.Add(ref bytesDownloaded, read); } } return bytesDownloaded; }
private static Action GetEncryptionAction( CancellationToken cancellationToken, FileEncryption fileEncryption, IngestManifestFileData file, string destinationPath, FileInfo fileInfo, ConcurrentQueue <Tuple <int, int> > queue, int maxBlockSize) { Action action = () => { cancellationToken.ThrowIfCancellationRequested(); if (queue.Count > 0) { using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read)) { Tuple <int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); var buffer = new byte[blockIdAndLength.Item2]; var binaryReader = new BinaryReader(fs); // Move the file system reader to the proper position. fs.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2); while (readSize != blockIdAndLength.Item2) { readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize); } bool lockWasTakenForEncode = false; try { Monitor.Enter(fileEncryption, ref lockWasTakenForEncode); using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1 * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } finally { if (lockWasTakenForEncode) { Monitor.Exit(fileEncryption); } } bool lockWasTakenForWrite = false; try { Monitor.Enter(file, ref lockWasTakenForWrite); using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write)) { writeStream.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin); writeStream.Write(buffer, 0, readSize); } } finally { if (lockWasTakenForWrite) { Monitor.Exit(file); } } } } } }; return(action); }
private static Action GetEncryptionAction( CancellationToken cancellationToken, FileEncryption fileEncryption, IngestManifestFileData file, string destinationPath, FileInfo fileInfo, ConcurrentQueue<Tuple<int, int>> queue, int maxBlockSize) { Action action = () => { cancellationToken.ThrowIfCancellationRequested(); if (queue.Count > 0) { using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read)) { Tuple<int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); var buffer = new byte[blockIdAndLength.Item2]; var binaryReader = new BinaryReader(fs); // Move the file system reader to the proper position. fs.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2); while (readSize != blockIdAndLength.Item2) { readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize); } bool lockWasTakenForEncode = false; try { Monitor.Enter(fileEncryption, ref lockWasTakenForEncode); using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1*(long) maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } finally { if (lockWasTakenForEncode) Monitor.Exit(fileEncryption); } bool lockWasTakenForWrite = false; try { Monitor.Enter(file, ref lockWasTakenForWrite); using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write)) { writeStream.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin); writeStream.Write(buffer, 0, readSize); } } finally { if (lockWasTakenForWrite) Monitor.Exit(file); } } } } }; return action; }
private void DownloadFileFromBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature, bool shouldDoFileIO = true, long start = 0, long length = -1, int parallelTransferThreadCount = 10) { ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy }; CloudBlockBlob blob = null; BlobTransferContext transferContext = new BlobTransferContext(); transferContext.Exceptions = new ConcurrentBag <Exception>(); try { blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature); long initialOffset = start; long sizeToDownload = blob.Properties.Length; if (length != -1) { if (length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length)); } if (start + length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length)); } sizeToDownload = length; } transferContext.Length = sizeToDownload; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => downloadCompletedSignal.Set(); transferContext.Blob = blob; transferContext.FileEncryption = fileEncryption; transferContext.InitializationVector = initializationVector; transferContext.InitialOffset = start; if (sizeToDownload == 0) { using (FileStream stream = new FileStream( localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { } } else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit) { AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition(); OperationContext operationContext = new OperationContext(); operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (FileStream fileStream = new FileStream( transferContext.LocalFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read )) { blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext); if (fileEncryption != null) { using (MemoryStream msDecrypt = new MemoryStream()) { //Using CryptoTransform APIs per Quintin's suggestion. using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0)) { fileStream.Position = 0; fileStream.CopyTo(msDecrypt); msDecrypt.Position = 0; fileStream.Position = 0; using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read)) { csEncrypt.CopyTo(fileStream); } } } } } InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload); transferContext.OnComplete(); } else { int numThreads = parallelTransferThreadCount; int blockSize = GetBlockSize(blob.Properties.Length); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset); transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (FileStream stream = new FileStream( transferContext.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { stream.SetLength(sizeToDownload); RunDownloadLoop(transferContext, stream, numThreads); } } } catch (Exception e) { //Add the exception to the exception list. transferContext.Exceptions.Add(e); } finally { // We should to be able to releaseunusedbuffers if memorymanager was initialized by then if (transferContext.MemoryManager != null) { transferContext.MemoryManager.ReleaseUnusedBuffers(); } //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user. TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Download, localFile, uri); } }
private void DownloadFileFromBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature, bool shouldDoFileIO = true, long start = 0, long length = -1, int parallelTransferThreadCount = 10) { ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy }; CloudBlockBlob blob = null; BlobTransferContext transferContext = new BlobTransferContext(); transferContext.Exceptions = new ConcurrentBag<Exception>(); try { blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature); long initialOffset = start; long sizeToDownload = blob.Properties.Length; if (length != -1) { if (length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length)); } if (start + length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length)); } sizeToDownload = length; } transferContext.Length = sizeToDownload; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => downloadCompletedSignal.Set(); transferContext.Blob = blob; transferContext.FileEncryption = fileEncryption; transferContext.InitializationVector = initializationVector; transferContext.InitialOffset = start; if (sizeToDownload == 0) { using (FileStream stream = new FileStream( localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { } } else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit) { AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition(); OperationContext operationContext = new OperationContext(); operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (FileStream fileStream = new FileStream( transferContext.LocalFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read )) { blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext); if (fileEncryption != null) { using (MemoryStream msDecrypt = new MemoryStream()) { //Using CryptoTransform APIs per Quintin's suggestion. using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0)) { fileStream.Position = 0; fileStream.CopyTo(msDecrypt); msDecrypt.Position = 0; fileStream.Position = 0; using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read)) { csEncrypt.CopyTo(fileStream); } } } } } InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload); transferContext.OnComplete(); } else { int numThreads = parallelTransferThreadCount; int blockSize = GetBlockSize(blob.Properties.Length); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset); transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (FileStream stream = new FileStream( transferContext.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { stream.SetLength(sizeToDownload); RunDownloadLoop(transferContext, stream, numThreads); } } } catch(Exception e) { //Add the exception to the exception list. transferContext.Exceptions.Add(e); } finally { // We should to be able to releaseunusedbuffers if memorymanager was initialized by then if (transferContext.MemoryManager != null) { transferContext.MemoryManager.ReleaseUnusedBuffers(); } //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user. TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Download, localFile, uri); } }