private void RunUploadLoop(
BlobTransferContext transferContext,
FileStream fileStream,
int numThreads)
{
SpinWait spinWait = new SpinWait();
while (!transferContext.IsComplete && !transferContext.CancellationToken.IsCancellationRequested)
{
if (!transferContext.IsReadingOrWriting)
{
DoSequentialRead(transferContext, fileStream);
}
if (!transferContext.IsComplete &&
transferContext.NumInProgressUploadDownloads < numThreads)
{
TryUploadingBlocks(transferContext);
}
spinWait.SpinOnce();
}
while (transferContext.NumInProgressUploadDownloads > 0 || transferContext.IsReadingOrWriting)
{
spinWait.SpinOnce();
}
foreach (var memoryStream in transferContext.BufferStreams.Values)
{
memoryStream.Dispose();
}
transferContext.OnComplete();
}
private void BeginUploadStream(
BlobTransferContext transferContext,
KeyValuePair <long, int> startAndLength,
MemoryStream memoryStream,
byte[] streamBuffer)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
memoryStream.Seek(0, SeekOrigin.Begin);
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
Interlocked.Increment(ref transferContext.NumInProgressUploadDownloads);
string blockId =
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0:d7}", (startAndLength.Key / transferContext.BlockSize))));
memoryStream.SetLength(startAndLength.Value);
transferContext.Blob.BeginPutBlock(
blockId,
memoryStream,
null,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndPutBlock(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginUploadStream(transferContext, startAndLength, memoryStream, streamBuffer);
return;
}
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
if (!wasWriteSuccessful.IsSuccessful)
{
return;
}
Interlocked.Add(ref transferContext.BytesBlobIOCompleted, startAndLength.Value);
InvokeProgressCallback(transferContext, transferContext.BytesBlobIOCompleted, startAndLength.Value);
if (transferContext.BytesBlobIOCompleted >= transferContext.Length)
{
BeginPutBlockList(transferContext);
}
},
state: null);
}
private void TryDownloadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
byte[] streamBuffer = transferContext.MemoryManager.RequireBuffer();
if (streamBuffer == null)
{
return;
}
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
KeyValuePair <long, int> startAndLength;
if (transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
}
private void RunUploadLoop(
BlobTransferContext transferContext,
Stream fileStream,
int numThreads)
{
SpinWait spinWait = new SpinWait();
while (!transferContext.IsComplete && !transferContext.CancellationToken.IsCancellationRequested)
{
if (!transferContext.IsReadingOrWriting)
{
DoSequentialRead(transferContext, fileStream);
}
if (!transferContext.IsComplete &&
transferContext.NumInProgressUploadDownloads < numThreads)
{
TryUploadingBlocks(transferContext);
}
spinWait.SpinOnce();
}
while (transferContext.NumInProgressUploadDownloads > 0 || transferContext.IsReadingOrWriting)
{
spinWait.SpinOnce();
}
//Release any buffers that are still in queue to be written to file but could not because there was
// a complete signal for this file upload due to some error in the one of the other block uploads in the same transfer context.
//If this is not cleaned, used buffers will hit the cap of 16 and future uploads will hang for lack of memory buffers.
for (int currentBlock = transferContext.NextFileIOBlock; currentBlock <= transferContext.BlocksForFileIO.Count(); currentBlock++)
{
byte[] buffer = null;
if (transferContext.BlocksForFileIO.TryGetValue(currentBlock, out buffer) && (buffer != null))
{
try
{
transferContext.MemoryManager.ReleaseBuffer(buffer);
}
catch (ArgumentException ex)
{
Debug.WriteLine("Exception occured while releasing memory buffer ", ex.Message);
}
}
}
foreach (var memoryStream in transferContext.BufferStreams.Values)
{
memoryStream.Dispose();
}
transferContext.OnComplete();
}
protected SuccessfulOrRetryableResult IsActionSuccessfulOrRetryable <T>(
BlobTransferContext transferContext,
Func <T> action,
out T returnValue)
{
T value = default(T);
SuccessfulOrRetryableResult result =
IsActionSuccessfulOrRetryable(
transferContext,
() => value = action());
returnValue = value;
return(result);
}
private void BeginPutBlockList(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
List <string> blockids = new List <string>();
for (int i = 0; i < (transferContext.Length + transferContext.BlockSize - 1) / transferContext.BlockSize; i++)
{
blockids.Add(
Convert.ToBase64String(
Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0:d7}", i))));
}
transferContext.Blob.BeginPutBlockList(
blockids,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndPutBlockList(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginPutBlockList(transferContext);
return;
}
transferContext.IsComplete = true;
},
state: null);
}
protected void InvokeProgressCallback(BlobTransferContext transferContext, long bytesProcessed, long lastBlockSize)
{
if (transferContext == null)
{
throw new ArgumentNullException("transferContext");
}
int progress = (int)((double)bytesProcessed / transferContext.Length * 100);
double speed = _uploadDownloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesProcessed);
BlobTransferProgressChangedEventArgs eArgs = new BlobTransferProgressChangedEventArgs(
bytesProcessed,
lastBlockSize,
transferContext.Length,
progress,
speed,
transferContext.Blob.Uri,
transferContext.LocalFilePath,
null);
OnTaskProgressChanged(eArgs);
}
private void TryUploadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (transferContext.NextFileIOBlock >= transferContext.BlocksForFileIO.Count)
{
return;
}
byte[] streamBuffer;
int nextBlock = transferContext.NextFileIOBlock;
if (transferContext.BlocksForFileIO.TryGetValue(nextBlock, out streamBuffer) &&
streamBuffer != null)
{
Interlocked.Increment(ref transferContext.NextFileIOBlock);
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
long beginFilePosition = (long)nextBlock * transferContext.BlockSize;
long nextBeginFilePosition = beginFilePosition + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
BeginUploadStream(
transferContext,
new KeyValuePair <long, int>(beginFilePosition, bytesToRead),
memoryStream,
streamBuffer);
}
}
private void TryDownloadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
//This is where memory is allocated, any code after this should make sure releasebuffer happens on all positive/negative conditions
byte[] streamBuffer = transferContext.MemoryManager.RequireBuffer();
if (streamBuffer == null)
{
return;
}
// Catch any exceptions and cleanup the buffer. Otherwise uncleaned buffers will result in hang for future
// downloads as memorymanager is being shared.
// Also mark the transfer as complete and add exceptions to the transfercontext exception list.
try
{
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
KeyValuePair <long, int> startAndLength;
if (transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
}
catch (Exception ex)
{
transferContext.IsComplete = true;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
transferContext.Exceptions.Add(ex);
}
}
private void DoSequentialWrite(
BlobTransferContext transferContext,
FileStream stream)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
byte[] buffer = null;
if (transferContext.BlocksForFileIO.TryGetValue(transferContext.NextFileIOBlock, out buffer) && buffer != null)
{
transferContext.IsReadingOrWriting = true;
long endOfRange = transferContext.Length + transferContext.InitialOffset;
long beginFilePosition = (long)transferContext.NextFileIOBlock * transferContext.BlockSize + transferContext.InitialOffset;
beginFilePosition = beginFilePosition > endOfRange
? endOfRange
: beginFilePosition;
long nextBeginFilePosition = (transferContext.NextFileIOBlock + 1) * (long)transferContext.BlockSize + transferContext.InitialOffset;
nextBeginFilePosition = nextBeginFilePosition > endOfRange
? endOfRange
: nextBeginFilePosition;
int bytesToWrite = (int)(nextBeginFilePosition - beginFilePosition);
ApplyEncryptionTransform(transferContext.FileEncryption, transferContext.InitializationVector, beginFilePosition, buffer, bytesToWrite);
stream.BeginWrite(
buffer,
0,
bytesToWrite,
result3 =>
{
SuccessfulOrRetryableResult wasWriteSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndWrite(result3));
transferContext.MemoryManager.ReleaseBuffer(buffer);
if (!wasWriteSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
return;
}
transferContext.NextFileIOBlock++;
Interlocked.Add(ref transferContext.BytesWrittenOrReadToFile, bytesToWrite);
InvokeProgressCallback(transferContext, transferContext.BytesWrittenOrReadToFile, bytesToWrite);
transferContext.IsReadingOrWriting = false;
if (transferContext.BytesWrittenOrReadToFile >= transferContext.Length)
{
transferContext.IsComplete = true;
transferContext.OnComplete();
}
},
null);
}
}
private void BeginUploadStream(
BlobTransferContext transferContext,
KeyValuePair<long, int> startAndLength,
MemoryStream memoryStream,
byte[] streamBuffer)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
memoryStream.Seek(0, SeekOrigin.Begin);
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
Interlocked.Increment(ref transferContext.NumInProgressUploadDownloads);
string blockId =
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0:d7}", (startAndLength.Key / transferContext.BlockSize))));
memoryStream.SetLength(startAndLength.Value);
transferContext.Blob.BeginPutBlock(
blockId,
memoryStream,
null,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndPutBlock(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginUploadStream(transferContext, startAndLength, memoryStream, streamBuffer);
return;
}
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
if (!wasWriteSuccessful.IsSuccessful)
{
return;
}
Interlocked.Add(ref transferContext.BytesBlobIOCompleted, startAndLength.Value);
InvokeProgressCallback(transferContext, transferContext.BytesBlobIOCompleted, startAndLength.Value);
if (transferContext.BytesBlobIOCompleted >= transferContext.Length)
{
BeginPutBlockList(transferContext);
}
},
state: null);
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1)
{
int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions {
RetryPolicy = retryPolicy
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length));
}
sizeToDownload = length;
}
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
null,
BlobTransferType.Download,
localFile,
uri);
}
else
{
int blockSize = GetBlockSize(blob.Properties.Length);
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.Exceptions = new ConcurrentBag <Exception>();
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
transferContext.MemoryManager.ReleaseUnusedBuffers();
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
BlobTransferType.Download,
localFile,
uri);
}
}
private void DoSequentialRead(
BlobTransferContext transferContext,
FileStream stream,
byte[] streamBuffer = null,
KeyValuePair <long, int>?inputStartAndLength = null)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (streamBuffer == null)
{
streamBuffer = transferContext.MemoryManager.RequireBuffer();
}
if (streamBuffer == null)
{
return;
}
KeyValuePair <long, int> startAndLength;
if (inputStartAndLength == null)
{
if (!transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
}
else
{
startAndLength = inputStartAndLength.Value;
}
if (!transferContext.PartialFileIOState.ContainsKey(startAndLength.Key))
{
transferContext.PartialFileIOState[startAndLength.Key] = 0;
}
transferContext.IsReadingOrWriting = true;
long beginFilePosition = startAndLength.Key;
long nextBeginFilePosition = startAndLength.Key + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
beginFilePosition = beginFilePosition + transferContext.PartialFileIOState[startAndLength.Key];
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
stream.BeginRead(
streamBuffer,
transferContext.PartialFileIOState[startAndLength.Key],
bytesToRead,
result3 =>
{
int bytesRead;
SuccessfulOrRetryableResult wasReadSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndRead(result3), out bytesRead);
if (!wasReadSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
else if (bytesRead != bytesToRead)
{
transferContext.PartialFileIOState[startAndLength.Key] += bytesRead;
DoSequentialRead(transferContext, stream, streamBuffer, startAndLength);
}
else
{
transferContext.IsReadingOrWriting = false;
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
beginFilePosition,
streamBuffer,
bytesToRead);
transferContext.BlocksForFileIO[(int)(startAndLength.Key / transferContext.BlockSize)] = streamBuffer;
}
},
null);
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func<string> getSharedAccessSignature,
int parallelTransferThreadCount,
bool shouldDoFileIO = true)
{
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag<Exception>();
try
{
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
//attempt to open the file first so that we throw an exception before getting into the async work
using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
}
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType,
getSharedAccessSignature);
BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));
FileInfo file = new FileInfo(localFile);
long fileSize = file.Length;
transferContext.Length = fileSize;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
if (fileSize == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
}
else if (fileSize < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (var memoryStream = new MemoryStream())
{
fileStream.CopyTo(memoryStream);
byte[] fileContent = memoryStream.ToArray();
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
0,
fileContent,
Convert.ToInt32(fileStream.Length));
using (var uploadMemoryStream = new MemoryStream(fileContent))
{
blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
}
}
}
InvokeProgressCallback(transferContext, fileSize, fileSize);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(fileSize);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>();
using (
FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read,
FileShare.ReadWrite))
{
RunUploadLoop(transferContext, stream, numThreads);
}
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Upload,
localFile,
uri);
}
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1,
int parallelTransferThreadCount = 10)
{
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions {
RetryPolicy = retryPolicy
};
CloudBlockBlob blob = null;
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag <Exception>();
try
{
blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
blob.Properties.Length));
}
sizeToDownload = length;
}
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
}
else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream fileStream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.ReadWrite,
FileShare.Read
))
{
blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
if (fileEncryption != null)
{
using (MemoryStream msDecrypt = new MemoryStream())
{
//Using CryptoTransform APIs per Quintin's suggestion.
using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0))
{
fileStream.Position = 0;
fileStream.CopyTo(msDecrypt);
msDecrypt.Position = 0;
fileStream.Position = 0;
using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read))
{
csEncrypt.CopyTo(fileStream);
}
}
}
}
}
InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(blob.Properties.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Download,
localFile,
uri);
}
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func<string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1,
int parallelTransferThreadCount = 10)
{
int numThreads = Environment.ProcessorCount * parallelTransferThreadCount;
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy };
CloudBlockBlob blob = null;
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag<Exception>();
try
{
blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
blob.Properties.Length));
}
sizeToDownload = length;
}
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
}
else
{
int blockSize = GetBlockSize(blob.Properties.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
}
}
catch(Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Download,
localFile,
uri);
}
}
private void RunUploadLoop(
BlobTransferContext transferContext,
FileStream fileStream,
int numThreads)
{
SpinWait spinWait = new SpinWait();
while (!transferContext.IsComplete && !transferContext.CancellationToken.IsCancellationRequested)
{
if (!transferContext.IsReadingOrWriting)
{
DoSequentialRead(transferContext, fileStream);
}
if (!transferContext.IsComplete &&
transferContext.NumInProgressUploadDownloads < numThreads)
{
TryUploadingBlocks(transferContext);
}
spinWait.SpinOnce();
}
while (transferContext.NumInProgressUploadDownloads > 0 || transferContext.IsReadingOrWriting)
{
spinWait.SpinOnce();
}
//Release any buffers that are still in queue to be written to file but could not because there was
// a complete signal for this file upload due to some error in the one of the other block uploads in the same transfer context.
//If this is not cleaned, used buffers will hit the cap of 16 and future uploads will hang for lack of memory buffers.
for (int currentBlock = transferContext.NextFileIOBlock; currentBlock <= transferContext.BlocksForFileIO.Count(); currentBlock++)
{
byte[] buffer = null;
if (transferContext.BlocksForFileIO.TryGetValue(currentBlock, out buffer) && (buffer != null))
{
try
{
transferContext.MemoryManager.ReleaseBuffer(buffer);
}
catch (ArgumentException ex)
{
Debug.WriteLine("Exception occured while releasing memory buffer ", ex.Message);
}
}
}
foreach (var memoryStream in transferContext.BufferStreams.Values)
{
memoryStream.Dispose();
}
transferContext.OnComplete();
}
private void BeginDownloadStream(
BlobTransferContext transferContext,
MemoryStream memoryStream,
KeyValuePair<long, int> startAndLength,
byte[] streamBuffer)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
memoryStream.Seek(0, SeekOrigin.Begin);
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
Interlocked.Increment(ref transferContext.NumInProgressUploadDownloads);
transferContext.Blob.BeginDownloadRangeToStream(
memoryStream,
startAndLength.Key,
startAndLength.Value,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndDownloadStream(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
return;
}
if (!wasWriteSuccessful.IsSuccessful)
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
Interlocked.Add(ref transferContext.BytesBlobIOCompleted, startAndLength.Value);
TryDownloadingBlocks(transferContext);
if (transferContext.ShouldDoFileIO)
{
transferContext.BlocksForFileIO[(int)(startAndLength.Key / transferContext.BlockSize)] = streamBuffer;
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
if (transferContext.BytesBlobIOCompleted >= transferContext.Length)
{
transferContext.IsComplete = true;
}
}
},
null);
}
private void DoSequentialWrite(
BlobTransferContext transferContext,
FileStream stream)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
byte[] buffer = null;
if (transferContext.BlocksForFileIO.TryGetValue(transferContext.NextFileIOBlock, out buffer) && buffer != null)
{
transferContext.IsReadingOrWriting = true;
long endOfRange = transferContext.Length + transferContext.InitialOffset;
long beginFilePosition = (long)transferContext.NextFileIOBlock * transferContext.BlockSize + transferContext.InitialOffset;
beginFilePosition = beginFilePosition > endOfRange
? endOfRange
: beginFilePosition;
long nextBeginFilePosition = (transferContext.NextFileIOBlock + 1) * (long)transferContext.BlockSize + transferContext.InitialOffset;
nextBeginFilePosition = nextBeginFilePosition > endOfRange
? endOfRange
: nextBeginFilePosition;
int bytesToWrite = (int)(nextBeginFilePosition - beginFilePosition);
ApplyEncryptionTransform(transferContext.FileEncryption, transferContext.InitializationVector, beginFilePosition, buffer, bytesToWrite);
stream.BeginWrite(
buffer,
0,
bytesToWrite,
result3 =>
{
SuccessfulOrRetryableResult wasWriteSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndWrite(result3));
transferContext.MemoryManager.ReleaseBuffer(buffer);
if (!wasWriteSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
return;
}
transferContext.NextFileIOBlock++;
Interlocked.Add(ref transferContext.BytesWrittenOrReadToFile, bytesToWrite);
InvokeProgressCallback(transferContext, transferContext.BytesWrittenOrReadToFile, bytesToWrite);
transferContext.IsReadingOrWriting = false;
if (transferContext.BytesWrittenOrReadToFile >= transferContext.Length)
{
transferContext.IsComplete = true;
transferContext.OnComplete();
}
},
null);
}
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func<string> getSharedAccessSignature,
bool shouldDoFileIO = true)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
}
SetConnectionLimits(uri);
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);
BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));
FileInfo file = new FileInfo(localFile);
long fileSize = file.Length;
if (fileSize == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
null,
BlobTransferType.Upload,
localFile,
uri);
}
else
{
int numThreads = Environment.ProcessorCount*ParallelUploadDownloadThreadCountMultiplier;
int blockSize = GetBlockSize(fileSize);
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = fileSize;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.Exceptions = new ConcurrentBag<Exception>();
transferContext.FileEncryption = fileEncryption;
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>();
using (
FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
RunUploadLoop(transferContext, stream, numThreads);
}
transferContext.MemoryManager.ReleaseUnusedBuffers();
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
BlobTransferType.Upload,
localFile,
uri);
}
}
private void RunUploadLoop(
BlobTransferContext transferContext,
FileStream fileStream,
int numThreads)
{
SpinWait spinWait = new SpinWait();
while (!transferContext.IsComplete && !transferContext.CancellationToken.IsCancellationRequested)
{
if (!transferContext.IsReadingOrWriting)
{
DoSequentialRead(transferContext, fileStream);
}
if (!transferContext.IsComplete &&
transferContext.NumInProgressUploadDownloads < numThreads)
{
TryUploadingBlocks(transferContext);
}
spinWait.SpinOnce();
}
while (transferContext.NumInProgressUploadDownloads > 0 || transferContext.IsReadingOrWriting)
{
spinWait.SpinOnce();
}
foreach (var memoryStream in transferContext.BufferStreams.Values)
{
memoryStream.Dispose();
}
transferContext.OnComplete();
}
private void DoSequentialRead(
BlobTransferContext transferContext,
FileStream stream,
byte[] streamBuffer = null,
KeyValuePair<long, int>? inputStartAndLength = null)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (streamBuffer == null)
{
streamBuffer = transferContext.MemoryManager.RequireBuffer();
}
if (streamBuffer == null)
{
return;
}
KeyValuePair<long, int> startAndLength;
if (inputStartAndLength == null)
{
if (!transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
}
else
{
startAndLength = inputStartAndLength.Value;
}
if (!transferContext.PartialFileIOState.ContainsKey(startAndLength.Key))
{
transferContext.PartialFileIOState[startAndLength.Key] = 0;
}
transferContext.IsReadingOrWriting = true;
long beginFilePosition = startAndLength.Key;
long nextBeginFilePosition = startAndLength.Key + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
beginFilePosition = beginFilePosition + transferContext.PartialFileIOState[startAndLength.Key];
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
stream.BeginRead(
streamBuffer,
transferContext.PartialFileIOState[startAndLength.Key],
bytesToRead,
result3 =>
{
int bytesRead;
SuccessfulOrRetryableResult wasReadSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndRead(result3), out bytesRead);
if (!wasReadSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
else if (bytesRead != bytesToRead)
{
transferContext.PartialFileIOState[startAndLength.Key] += bytesRead;
DoSequentialRead(transferContext, stream, streamBuffer, startAndLength);
}
else
{
transferContext.IsReadingOrWriting = false;
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
beginFilePosition,
streamBuffer,
bytesToRead);
transferContext.BlocksForFileIO[(int) (startAndLength.Key/transferContext.BlockSize)] = streamBuffer;
}
},
null);
}
private void DoSequentialRead(
BlobTransferContext transferContext,
FileStream stream,
byte[] streamBuffer = null,
KeyValuePair<long, int>? inputStartAndLength = null)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (streamBuffer == null)
{
streamBuffer = transferContext.MemoryManager.RequireBuffer();
}
if (streamBuffer == null)
{
return;
}
KeyValuePair<long, int> startAndLength;
if (inputStartAndLength == null)
{
if (!transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
}
else
{
startAndLength = inputStartAndLength.Value;
}
// Catch any exceptions and cleanup the buffer. Otherwise uncleaned buffers will result in hang for future
// uploads as memorymanager is being shared.
// Also mark the transfer as complete and add exceptions to the transfercontext exception list.
try
{
if (!transferContext.PartialFileIOState.ContainsKey(startAndLength.Key))
{
transferContext.PartialFileIOState[startAndLength.Key] = 0;
}
transferContext.IsReadingOrWriting = true;
long beginFilePosition = startAndLength.Key;
long nextBeginFilePosition = startAndLength.Key + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
beginFilePosition = beginFilePosition + transferContext.PartialFileIOState[startAndLength.Key];
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
stream.BeginRead(
streamBuffer,
transferContext.PartialFileIOState[startAndLength.Key],
bytesToRead,
result3 =>
{
int bytesRead;
SuccessfulOrRetryableResult wasReadSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndRead(result3), out bytesRead);
if (!wasReadSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
else if (bytesRead != bytesToRead)
{
transferContext.PartialFileIOState[startAndLength.Key] += bytesRead;
DoSequentialRead(transferContext, stream, streamBuffer, startAndLength);
}
else
{
transferContext.IsReadingOrWriting = false;
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
beginFilePosition,
streamBuffer,
bytesToRead);
transferContext.BlocksForFileIO[(int)(startAndLength.Key / transferContext.BlockSize)] = streamBuffer;
}
},
null);
}
catch (Exception ex)
{
transferContext.IsComplete = true;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
transferContext.Exceptions.Add(ex);
}
}
private void TryDownloadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
//This is where memory is allocated, any code after this should make sure releasebuffer happens on all positive/negative conditions
byte[] streamBuffer = transferContext.MemoryManager.RequireBuffer();
if (streamBuffer == null)
{
return;
}
// Catch any exceptions and cleanup the buffer. Otherwise uncleaned buffers will result in hang for future
// downloads as memorymanager is being shared.
// Also mark the transfer as complete and add exceptions to the transfercontext exception list.
try
{
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
KeyValuePair<long, int> startAndLength;
if (transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
}
catch (Exception ex)
{
transferContext.IsComplete = true;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
transferContext.Exceptions.Add(ex);
}
}
private SuccessfulOrRetryableResult EndPutBlockList(BlobTransferContext transferContext, IAsyncResult ar)
{
return IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndPutBlockList(ar));
}
protected virtual SuccessfulOrRetryableResult EndDownloadStream(BlobTransferContext transferContext, IAsyncResult ar)
{
return IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndDownloadRangeToStream(ar));
}
private void TryUploadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (transferContext.NextFileIOBlock >= transferContext.BlocksForFileIO.Count)
{
return;
}
byte[] streamBuffer;
int nextBlock = transferContext.NextFileIOBlock;
if (transferContext.BlocksForFileIO.TryGetValue(nextBlock, out streamBuffer)
&& streamBuffer != null)
{
Interlocked.Increment(ref transferContext.NextFileIOBlock);
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
long beginFilePosition = (long)nextBlock * transferContext.BlockSize;
long nextBeginFilePosition = beginFilePosition + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
BeginUploadStream(
transferContext,
new KeyValuePair<long, int>(beginFilePosition, bytesToRead),
memoryStream,
streamBuffer);
}
}
protected virtual SuccessfulOrRetryableResult EndPutBlock(BlobTransferContext transferContext, IAsyncResult ar)
{
return(IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndPutBlock(ar)));
}
protected virtual SuccessfulOrRetryableResult EndPutBlock(BlobTransferContext transferContext, IAsyncResult ar)
{
return IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndPutBlock(ar));
}
protected SuccessfulOrRetryableResult IsActionSuccessfulOrRetryable(BlobTransferContext transferContext, Action action)
{
if (transferContext == null)
{
throw new ArgumentNullException("transferContext");
}
if (action == null)
{
throw new ArgumentNullException("action");
}
SuccessfulOrRetryableResult result =
new SuccessfulOrRetryableResult
{
IsRetryable = false,
IsSuccessful = false
};
try
{
action();
}
catch (Exception exception)
{
WebException webEx = exception.FindInnerException <WebException>();
if (webEx == null)
{
transferContext.Exceptions.Add(exception);
transferContext.IsComplete = true;
return(result);
}
if (transferContext.GetSharedAccessSignature != null)
{
if (webEx.Response is HttpWebResponse)
{
var httpex = (HttpWebResponse)webEx.Response;
if (httpex.StatusCode == HttpStatusCode.Forbidden)
{
Interlocked.Increment(ref transferContext.SasRetryCount);
if (transferContext.SasRetryCount > MaxSasSignatureRetry)
{
transferContext.Exceptions.Add(exception);
transferContext.IsComplete = true;
return(result);
}
Thread.Sleep(SasSignatureRetryTime);
result.IsRetryable = true;
return(result);
}
}
}
transferContext.Exceptions.Add(exception);
transferContext.IsComplete = true;
return(result);
}
result.IsSuccessful = true;
return(result);
}
private void TryDownloadingBlocks(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
byte[] streamBuffer = transferContext.MemoryManager.RequireBuffer();
if (streamBuffer == null)
{
return;
}
MemoryStream memoryStream = GetMemoryStream(transferContext.BufferStreams, streamBuffer);
KeyValuePair<long, int> startAndLength;
if (transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
}
private void BeginDownloadStream(
BlobTransferContext transferContext,
MemoryStream memoryStream,
KeyValuePair <long, int> startAndLength,
byte[] streamBuffer)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
memoryStream.Seek(0, SeekOrigin.Begin);
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
Interlocked.Increment(ref transferContext.NumInProgressUploadDownloads);
transferContext.Blob.BeginDownloadRangeToStream(
memoryStream,
startAndLength.Key,
startAndLength.Value,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndDownloadStream(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginDownloadStream(transferContext, memoryStream, startAndLength, streamBuffer);
return;
}
if (!wasWriteSuccessful.IsSuccessful)
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
Interlocked.Add(ref transferContext.BytesBlobIOCompleted, startAndLength.Value);
TryDownloadingBlocks(transferContext);
if (transferContext.ShouldDoFileIO)
{
transferContext.BlocksForFileIO[(int)(startAndLength.Key / transferContext.BlockSize)] = streamBuffer;
}
else
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
if (transferContext.BytesBlobIOCompleted >= transferContext.Length)
{
transferContext.IsComplete = true;
}
}
},
null);
}
private void DoSequentialRead(
BlobTransferContext transferContext,
Stream stream,
byte[] streamBuffer = null,
KeyValuePair <long, int>?inputStartAndLength = null)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
if (streamBuffer == null)
{
streamBuffer = transferContext.MemoryManager.RequireBuffer();
}
if (streamBuffer == null)
{
return;
}
KeyValuePair <long, int> startAndLength;
if (inputStartAndLength == null)
{
if (!transferContext.BlocksToTransfer.TryDequeue(out startAndLength))
{
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
return;
}
}
else
{
startAndLength = inputStartAndLength.Value;
}
// Catch any exceptions and cleanup the buffer. Otherwise uncleaned buffers will result in hang for future
// uploads as memorymanager is being shared.
// Also mark the transfer as complete and add exceptions to the transfercontext exception list.
try
{
if (!transferContext.PartialFileIOState.ContainsKey(startAndLength.Key))
{
transferContext.PartialFileIOState[startAndLength.Key] = 0;
}
transferContext.IsReadingOrWriting = true;
long beginFilePosition = startAndLength.Key;
long nextBeginFilePosition = startAndLength.Key + transferContext.BlockSize;
nextBeginFilePosition =
nextBeginFilePosition > transferContext.Length
? transferContext.Length
: nextBeginFilePosition;
beginFilePosition = beginFilePosition + transferContext.PartialFileIOState[startAndLength.Key];
int bytesToRead = (int)(nextBeginFilePosition - beginFilePosition);
stream.BeginRead(
streamBuffer,
transferContext.PartialFileIOState[startAndLength.Key],
bytesToRead,
result3 =>
{
int bytesRead;
SuccessfulOrRetryableResult wasReadSuccessful =
IsActionSuccessfulOrRetryable(transferContext, () => stream.EndRead(result3), out bytesRead);
if (!wasReadSuccessful.IsSuccessful)
{
transferContext.IsReadingOrWriting = false;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
}
else if (bytesRead != bytesToRead)
{
transferContext.PartialFileIOState[startAndLength.Key] += bytesRead;
DoSequentialRead(transferContext, stream, streamBuffer, startAndLength);
}
else
{
transferContext.IsReadingOrWriting = false;
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
beginFilePosition,
streamBuffer,
bytesToRead);
transferContext.BlocksForFileIO[(int)(startAndLength.Key / transferContext.BlockSize)] = streamBuffer;
}
},
null);
}
catch (Exception ex)
{
transferContext.IsComplete = true;
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
transferContext.Exceptions.Add(ex);
}
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string name,
Stream stream,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
int parallelTransferThreadCount,
bool shouldDoFileIO = true)
{
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag <Exception>();
try
{
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, name, contentType, getSharedAccessSignature);
transferContext.Length = stream.Length;
transferContext.LocalFilePath = name;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
if (stream.Length == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
}
else if (stream.Length < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (var memoryStream = new MemoryStream())
{
stream.CopyTo(memoryStream);
byte[] fileContent = memoryStream.ToArray();
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
0,
fileContent,
Convert.ToInt32(stream.Length));
using (var uploadMemoryStream = new MemoryStream(fileContent))
{
blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
}
}
InvokeProgressCallback(transferContext, stream.Length, stream.Length);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(stream.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(stream.Length, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>();
RunUploadLoop(transferContext, stream, numThreads);
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Upload,
name,
uri);
}
}
private SuccessfulOrRetryableResult EndPutBlockList(BlobTransferContext transferContext, IAsyncResult ar)
{
return(IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndPutBlockList(ar)));
}
private void BeginPutBlockList(BlobTransferContext transferContext)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
List<string> blockids = new List<string>();
for (int i = 0; i < (transferContext.Length + transferContext.BlockSize - 1) / transferContext.BlockSize; i++)
{
blockids.Add(
Convert.ToBase64String(
Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0:d7}", i))));
}
transferContext.Blob.BeginPutBlockList(
blockids,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndPutBlockList(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginPutBlockList(transferContext);
return;
}
transferContext.IsComplete = true;
},
state: null);
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
}
SetConnectionLimits(uri);
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);
BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));
FileInfo file = new FileInfo(localFile);
long fileSize = file.Length;
if (fileSize == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
null,
BlobTransferType.Upload,
localFile,
uri);
}
else
{
int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
int blockSize = GetBlockSize(fileSize);
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = fileSize;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.Exceptions = new ConcurrentBag <Exception>();
transferContext.FileEncryption = fileEncryption;
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>();
using (
FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
RunUploadLoop(transferContext, stream, numThreads);
}
transferContext.MemoryManager.ReleaseUnusedBuffers();
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
BlobTransferType.Upload,
localFile,
uri);
}
}
protected virtual SuccessfulOrRetryableResult EndDownloadStream(BlobTransferContext transferContext, IAsyncResult ar)
{
return(IsActionSuccessfulOrRetryable(transferContext, () => transferContext.Blob.EndDownloadRangeToStream(ar)));
}
