private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response,
KeyValuePair <long, int> blockOffsetAndLength, ref long bytesDownloaded)
{
using (Stream stream = response.GetResponseStream())
{
int offsetInChunk = 0;
int remaining = blockOffsetAndLength.Value;
while (remaining > 0)
{
int read = stream.Read(buffer, offsetInChunk, remaining);
lock (lockobject)
{
fs.Position = blockOffsetAndLength.Key + offsetInChunk;
if (fileEncryption != null)
{
lock (fileEncryption)
{
using (
FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk)
)
{
encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk);
}
}
}
fs.Write(buffer, offsetInChunk, read);
}
offsetInChunk += read;
remaining -= read;
Interlocked.Add(ref bytesDownloaded, read);
}
}
return(bytesDownloaded);
}
protected void ApplyEncryptionTransform(FileEncryption fileEncryption, string fileName, long beginFilePosition, byte[] buffer, int bytesToWrite)
{
if (fileEncryption != null)
{
lock (fileEncryption)
{
using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(fileName, beginFilePosition))
{
encryptor.TransformBlock(inputBuffer: buffer, inputOffset: 0, inputCount: bytesToWrite, outputBuffer: buffer, outputOffset: 0);
}
}
}
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subFolder,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
{
}
Exception lastException = null;
CloudBlockBlob blob = null;
// stats from azurescope show 10 to be an optimal number of transfer threads
int numThreads = ParallelTransferThreadCount;
var file = new FileInfo(localFile);
long fileSize = file.Length;
int maxBlockSize = GetBlockSize(fileSize);
// Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
// the 'key' is block id and 'value' is the block length.
List <string> blockList;
var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
int exceptionCount = 0;
blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
blob.DeleteIfExists(options: new BlobRequestOptions()
{
RetryPolicy = retryPolicy
});
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
var options = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
// Launch threads to upload blocks.
var tasks = new List <Task>();
long bytesSent = 0;
Action action = () =>
{
List <Exception> exceptions = new List <Exception>();
int sasRetry = 0;
if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
{
Thread.Sleep(_forceSharedAccessSignatureRetry);
}
if (queue.Count > 0)
{
FileStream fs = null;
try
{
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
KeyValuePair <int, int> blockIdAndLength;
while (queue.TryDequeue(out blockIdAndLength))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var buffer = new byte[blockIdAndLength.Value];
var binaryReader = new BinaryReader(fs);
// move the file system reader to the proper position
fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);
if (fileEncryption != null)
{
lock (fileEncryption)
{
using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
{
encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
}
}
}
using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
{
string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
string blockHash = GetMd5HashFromStream(buffer);
if (blob != null)
{
blob.PutBlock(blockIdString, ms, blockHash, options: options);
}
}
Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
var progress = (int)((double)bytesSent / file.Length * 100);
var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
OnTaskProgressChanged(eArgs);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
{
sasRetry++;
if (sasRetry > MaxSasSignatureRetry)
{
throw;
}
blob = GetCloudBlockBlob(uri, client, subFolder, localFile,
contentType, getSharedAccessSignature);
}
else
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
throw lastException;
}
Thread.Sleep(tm);
}
queue.Enqueue(blockIdAndLength);
}
catch (IOException ex)
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
throw lastException;
}
// dispose existing file stream
if (fs != null)
{
fs.Close();
}
Thread.Sleep(tm);
// try to reopen the file stream again
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
queue.Enqueue(blockIdAndLength);
}
}
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
};
for (int idxThread = 0; idxThread < numThreads; idxThread++)
{
tasks.Add(Task.Factory.StartNew(
action,
cancellationToken,
TaskCreationOptions.AttachedToParent,
TaskScheduler.Current));
}
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
{
if (result.Any(t => t.IsFaulted))
{
return;
}
try
{
blob.PutBlockList(blockList, options: options);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
{
blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
blob.PutBlockList(blockList, options: options);
}
else
{
throw;
}
}
}, TaskContinuationOptions.None).Wait(cancellationToken);
TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
}
private static Action GetEncryptionAction(
CancellationToken cancellationToken,
FileEncryption fileEncryption,
IngestManifestFileData file,
string destinationPath,
FileInfo fileInfo,
ConcurrentQueue <Tuple <int, int> > queue,
int maxBlockSize)
{
Action action = () =>
{
cancellationToken.ThrowIfCancellationRequested();
if (queue.Count > 0)
{
using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read))
{
Tuple <int, int> blockIdAndLength;
while (queue.TryDequeue(out blockIdAndLength))
{
cancellationToken.ThrowIfCancellationRequested();
var buffer = new byte[blockIdAndLength.Item2];
var binaryReader = new BinaryReader(fs);
// Move the file system reader to the proper position.
fs.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2);
while (readSize != blockIdAndLength.Item2)
{
readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize);
}
bool lockWasTakenForEncode = false;
try
{
Monitor.Enter(fileEncryption, ref lockWasTakenForEncode);
using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1 * (long)maxBlockSize))
{
encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
}
}
finally
{
if (lockWasTakenForEncode)
{
Monitor.Exit(fileEncryption);
}
}
bool lockWasTakenForWrite = false;
try
{
Monitor.Enter(file, ref lockWasTakenForWrite);
using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write))
{
writeStream.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
writeStream.Write(buffer, 0, readSize);
}
}
finally
{
if (lockWasTakenForWrite)
{
Monitor.Exit(file);
}
}
}
}
}
};
return(action);
}
private void DownloadFileFromBlob(
Uri uri,
string localFile,
FileEncryption fileEncryption,
ulong initializationVector,
CloudBlobClient client,
CancellationToken cancellationToken,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature,
bool shouldDoFileIO = true,
long start = 0,
long length = -1,
int parallelTransferThreadCount = 10)
{
ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions {
RetryPolicy = retryPolicy
};
CloudBlockBlob blob = null;
BlobTransferContext transferContext = new BlobTransferContext();
transferContext.Exceptions = new ConcurrentBag <Exception>();
try
{
blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long initialOffset = start;
long sizeToDownload = blob.Properties.Length;
if (length != -1)
{
if (length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
}
if (start + length > blob.Properties.Length)
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
"Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
blob.Properties.Length));
}
sizeToDownload = length;
}
transferContext.Length = sizeToDownload;
transferContext.LocalFilePath = localFile;
transferContext.OnComplete = () => downloadCompletedSignal.Set();
transferContext.Blob = blob;
transferContext.FileEncryption = fileEncryption;
transferContext.InitializationVector = initializationVector;
transferContext.InitialOffset = start;
if (sizeToDownload == 0)
{
using (FileStream stream =
new FileStream(
localFile,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
}
}
else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit)
{
AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream fileStream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.ReadWrite,
FileShare.Read
))
{
blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
if (fileEncryption != null)
{
using (MemoryStream msDecrypt = new MemoryStream())
{
//Using CryptoTransform APIs per Quintin's suggestion.
using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0))
{
fileStream.Position = 0;
fileStream.CopyTo(msDecrypt);
msDecrypt.Position = 0;
fileStream.Position = 0;
using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read))
{
csEncrypt.CopyTo(fileStream);
}
}
}
}
}
InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload);
transferContext.OnComplete();
}
else
{
int numThreads = parallelTransferThreadCount;
int blockSize = GetBlockSize(blob.Properties.Length);
transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
ref numThreads, initialOffset);
transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
{
transferContext.BlocksForFileIO[i] = null;
}
transferContext.BlockSize = blockSize;
transferContext.CancellationToken = cancellationToken;
transferContext.BlobRequestOptions = blobRequestOptions;
transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
transferContext.Client = client;
transferContext.RetryPolicy = retryPolicy;
transferContext.GetSharedAccessSignature = getSharedAccessSignature;
transferContext.ShouldDoFileIO = shouldDoFileIO;
transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>();
transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
using (FileStream stream = new FileStream(
transferContext.LocalFilePath,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.Read
))
{
stream.SetLength(sizeToDownload);
RunDownloadLoop(transferContext, stream, numThreads);
}
}
}
catch (Exception e)
{
//Add the exception to the exception list.
transferContext.Exceptions.Add(e);
}
finally
{
// We should to be able to releaseunusedbuffers if memorymanager was initialized by then
if (transferContext.MemoryManager != null)
{
transferContext.MemoryManager.ReleaseUnusedBuffers();
}
//TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
TaskCompletedCallback(
cancellationToken.IsCancellationRequested,
transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
? new AggregateException(transferContext.Exceptions)
: null,
BlobTransferType.Download,
localFile,
uri);
}
}