protected BlobTransferBase(MemoryManagerFactory memoryManagerFactory)
{
if (memoryManagerFactory == null)
{
throw new ArgumentNullException("memoryManagerFactory");
}
MemoryManagerFactory = memoryManagerFactory;
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
}
SetConnectionLimits(uri);
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);
BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));
FileInfo file = new FileInfo(localFile);
long fileSize = file.Length;
if (fileSize == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
null,
BlobTransferType.Upload,
localFile,
uri);
}
else
{
int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
int blockSize = GetBlockSize(fileSize);
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = fileSize;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.Exceptions = new ConcurrentBag <Exception>();
transferContext.FileEncryption = fileEncryption;
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>();
using (
FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
RunUploadLoop(transferContext, stream, numThreads);
}
transferContext.MemoryManager.ReleaseUnusedBuffers();
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
BlobTransferType.Upload,
localFile,
uri);
}
}
public BlobUploader(MemoryManagerFactory memoryManagerFactory) : base(memoryManagerFactory)
{
}
public BlobUploader(MemoryManagerFactory memoryManagerFactory)
: base(memoryManagerFactory)
{
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string name,
Stream stream,
string contentType,
string subDirectory,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
int parallelTransferThreadCount,
bool shouldDoFileIO = true)
{
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag <Exception>();
try
{
ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, name, contentType, getSharedAccessSignature);
transferContext.Length = stream.Length;
transferContext.LocalFilePath = name;
transferContext.OnComplete = () => uploadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
if (stream.Length == 0)
{
blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
}
else if (stream.Length < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (var memoryStream = new MemoryStream())
{
stream.CopyTo(memoryStream);
byte[] fileContent = memoryStream.ToArray();
ApplyEncryptionTransform(
transferContext.FileEncryption,
Path.GetFileName(transferContext.LocalFilePath),
0,
fileContent,
Convert.ToInt32(stream.Length));
using (var uploadMemoryStream = new MemoryStream(fileContent))
{
blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
}
}
InvokeProgressCallback(transferContext, stream.Length, stream.Length);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(stream.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(stream.Length, blockSize, ref numThreads);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.ContentType = contentType;
transferContext.BlobSubFolder = subDirectory;
transferContext.NextFileIOBlock = 0;
transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>();
RunUploadLoop(transferContext, stream, numThreads);
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Upload,
name,
uri);
}
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1)
{
int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions {
RetryPolicy = retryPolicy
};
CloudBlockBlob blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length));
}
sizeToDownload = length;
}
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
null,
BlobTransferType.Download,
localFile,
uri);
}
else
{
int blockSize = GetBlockSize(blob.Properties.Length);
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.Blob = blob;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
transferContext.Exceptions = new ConcurrentBag <Exception>();
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
transferContext.MemoryManager.ReleaseUnusedBuffers();
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
BlobTransferType.Download,
localFile,
uri);
}
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1,
int parallelTransferThreadCount = 10)
{
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions {
RetryPolicy = retryPolicy
};
CloudBlockBlob blob = null;
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag <Exception>();
try
{
blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
blob.Properties.Length));
}
sizeToDownload = length;
}
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
}
else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream fileStream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.ReadWrite,
FileShare.Read
))
{
blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
if (fileEncryption != null)
{
using (MemoryStream msDecrypt = new MemoryStream())
{
//Using CryptoTransform APIs per Quintin's suggestion.
using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0))
{
fileStream.Position = 0;
fileStream.CopyTo(msDecrypt);
msDecrypt.Position = 0;
fileStream.Position = 0;
using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read))
{
csEncrypt.CopyTo(fileStream);
}
}
}
}
}
InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(blob.Properties.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Download,
localFile,
uri);
}
}