/// <summary>
/// Rety an async operation based on the retry strategy supplied.
/// </summary>
/// <param name="asyncOperation">The async operation to be retried.</param>
/// <param name="retryPolicy">The retry policy to be applied.</param>
/// <param name="retryableExceptions">The exceptions to be retried on.</param>
/// <param name="logger">The <see cref="MsTestLogger"/> instance to be used.</param>
/// <returns></returns>
public static async Task RetryOperationsAsync(Func <Task> asyncOperation, IRetryPolicy retryPolicy, HashSet <Type> retryableExceptions, MsTestLogger logger)
{
int counter = 0;
bool shouldRetry;
do
{
TimeSpan retryInterval;
try
{
await asyncOperation().ConfigureAwait(false);
break;
}
catch (Exception ex) when(retryableExceptions.Any(e => e.IsInstanceOfType(ex)))
{
shouldRetry = retryPolicy.ShouldRetry(++counter, ex, out retryInterval);
logger.Trace($"Attempt {counter}: operation did not succeed: {ex}");
if (!shouldRetry)
{
logger.Trace($"Encountered an exception that will not be retried - attempt: {counter}; exception: {ex}");
throw;
}
}
logger.Trace($"Will retry operation in {retryInterval}.");
await Task.Delay(retryInterval).ConfigureAwait(false);
}while (shouldRetry);
}
/// <summary>
/// Adds retry behaviour to observable streams
/// </summary>
public static IObservable <T> RetryWithPolicy <T>(this IObservable <T> source, IRetryPolicy retryPolicy, string operationDescription)
{
return(Observable.Create <T>(o =>
{
int retryCount = 0;
Action subscribe = null;
SerialDisposable disposable = new SerialDisposable();
subscribe = () =>
{
disposable.Disposable = source.Subscribe(
o.OnNext,
ex => {
if (retryPolicy.ShouldRetry(ex, ++retryCount))
{
Debug.WriteLine("Retrying [{0}]. This is attempt [{1}]. Exception: [{2}]", operationDescription, retryCount, ex.Message);
subscribe();
}
else
{
Debug.WriteLine("Not retrying [{0}]. Retry count [{1}]. Will error. Exception: [{2}]", operationDescription, retryCount, ex.Message);
o.OnError(ex);
}
},
o.OnCompleted
);
};
subscribe();
return disposable;
}));
}
public async Task <U> RetryAsync <U>(Func <CloudTable <T>, Task <U> > func, IRetryPolicy policy = null)
{
policy ??= CloudTableContext.ServiceClient.DefaultRequestOptions.RetryPolicy;
var retry = 0;
while (true)
{
try
{
return(await func(this));
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.PreconditionFailed &&
e.RequestInformation.HttpStatusCode != (int)HttpStatusCode.Conflict)
{
throw;
}
if (!policy.ShouldRetry(retry++, 0, e, out TimeSpan delay, null))
{
throw;
}
await Task.Delay(delay);
}
}
}
public static async Task <T> RetryAsync <T>(this IRetryPolicy retryPolicy,
RetryDelegateAsync <T> callback,
Func <T> onFailureCallback,
int retryHttpStatus = 200, Exception retryException = default(Exception))
{
if (default(Exception) == retryException)
{
retryException = new Exception();
}
var retriesAttempted = 0;
TimeSpan retryDelay;
var result = default(T);
while (retryPolicy.ShouldRetry(retriesAttempted++, retryHttpStatus, retryException, out retryDelay, null))
{
var unlockSucceeded = false;
await callback((updatedResult) =>
{
unlockSucceeded = true;
result = updatedResult;
});
if (unlockSucceeded)
{
return(result);
}
await Task.Delay(retryDelay);
}
return(onFailureCallback());
}
public bool ShouldRetry(int currentRetryCount, Exception lastException, out TimeSpan retryInterval)
{
retryInterval = TimeSpan.Zero;
var provisioningException = lastException as ProvisioningServiceClientHttpException;
if (provisioningException == null || currentRetryCount > MaxRetryCount)
{
return(false);
}
else if ((int)provisioningException.StatusCode == 429) // HttpStatusCode.TooManyRequests is not available in net472
{
IDictionary <string, string> httpHeaders = provisioningException.Fields;
bool retryAfterPresent = httpHeaders.TryGetValue(RetryAfterKey, out string retryAfter);
retryInterval = retryAfterPresent
? TimeSpan.FromSeconds(Convert.ToDouble(retryAfter))
: s_defaultRetryInterval;
return(true);
}
else if ((int)provisioningException.StatusCode > 500 && (int)provisioningException.StatusCode < 600)
{
return(s_exponentialBackoffRetryStrategy.ShouldRetry(currentRetryCount, lastException, out retryInterval));
}
return(false);
}
private void VerifyBackoffTimeOverflow(IRetryPolicy retryPolicy, int maxAttempts)
{
StorageException e = new StorageException();
OperationContext context = new OperationContext();
TimeSpan retryInterval;
TimeSpan previousRetryInterval = TimeSpan.FromMilliseconds(1); // larger than zero to ensure we never get zero back
for (int i = 0; i < maxAttempts; i++)
{
Assert.IsTrue(retryPolicy.ShouldRetry(i, (int)HttpStatusCode.InternalServerError, e, out retryInterval, context), string.Format("Attempt: {0}", i));
Assert.IsTrue(retryInterval >= previousRetryInterval, string.Format("Retry Interval: {0}, Previous Retry Interval: {1}, Attempt: {2}", retryInterval, previousRetryInterval, i));
previousRetryInterval = retryInterval;
}
Assert.IsFalse(retryPolicy.ShouldRetry(maxAttempts, (int)HttpStatusCode.InternalServerError, e, out retryInterval, context));
}
public static T Retry <T>(this IRetryPolicy retryPolicy, Func <T> func)
{
int retryCount = 0;
while (true)
{
try
{
return(func());
}
catch (Exception ex)
{
TimeSpan?interval = retryPolicy.ShouldRetry(retryCount, ex);
if (interval.HasValue)
{
Thread.Sleep(interval.Value);
}
else
{
throw;
}
}
retryCount++;
}
}
public bool ShouldRetry(int currentRetryCount, int statusCode, Exception lastException, out TimeSpan retryInterval, Microsoft.WindowsAzure.Storage.OperationContext operationContext)
{
retryInterval = TimeSpan.FromSeconds(1);
if (lastException is OperationCanceledException || lastException.InnerException is OperationCanceledException)
{
return(false);
}
return(originalRetryPolicy.ShouldRetry(currentRetryCount, statusCode, lastException, out retryInterval, operationContext));
}
public bool ShouldRetry(int currentRetryCount, int statusCode, Exception lastException, out TimeSpan retryInterval, OperationContext operationContext)
{
if (statusCode == (int)HttpStatusCode.NotModified)
{
retryInterval = TimeSpan.Zero;
return(false);
}
else
{
return(_innerPolicy.ShouldRetry(currentRetryCount, statusCode, lastException, out retryInterval, operationContext));
}
}
private static CloudBlockBlob InitializeCloudBlockBlob(Uri uri, CloudBlobClient client, IRetryPolicy retryPolicy, Func <String> getSharedAccessSignature)
{
CloudBlockBlob blob = null;
if (getSharedAccessSignature != null)
{
string signature = getSharedAccessSignature();
blob = new CloudBlockBlob(uri, new StorageCredentials(signature));
}
else
{
if (client != null)
{
blob = new CloudBlockBlob(uri, client.Credentials);
}
else
{
blob = new CloudBlockBlob(uri);
}
}
bool fetch = false;
bool shouldRetry = true;
TimeSpan delay;
int retryCount = 0;
StorageException lastException = null;
while (!fetch && shouldRetry)
{
try
{
blob.FetchAttributes(options: new BlobRequestOptions()
{
RetryPolicy = retryPolicy
});
fetch = true;
}
catch (StorageException ex)
{
retryCount++;
lastException = ex;
shouldRetry = retryPolicy.ShouldRetry(retryCount, ex.RequestInformation.HttpStatusCode, lastException, out delay, new OperationContext());
Thread.Sleep(delay);
}
}
if (!fetch)
{
throw lastException;
}
return(blob);
}
bool ShouldRetry(int retryCount, Exception lastException, out TimeSpan retryInterval)
{
try
{
if (Logging.IsEnabled)
{
Logging.Enter(this, retryCount, lastException, $"{nameof(RetryStrategyAdapter)}.{nameof(ShouldRetry)}");
}
return(_retryStrategy.ShouldRetry(retryCount, lastException, out retryInterval));
}
finally
{
if (Logging.IsEnabled)
{
Logging.Exit(this, $"{nameof(RetryStrategyAdapter)}.{nameof(ShouldRetry)}");
}
}
}
private bool ShouldRetry(int retryCount, Exception lastException, out TimeSpan retryInterval)
{
bool shouldRetry = false;
try
{
if (Logging.IsEnabled)
{
Logging.Enter(this, retryCount, lastException, $"{nameof(RetryStrategyAdapter)}.{nameof(ShouldRetry)}");
}
shouldRetry = _retryStrategy.ShouldRetry(retryCount, lastException, out retryInterval);
Logging.Info(this, $"ShouldRetry = {shouldRetry}, after {retryInterval}");
return(shouldRetry);
}
finally
{
if (Logging.IsEnabled)
{
Logging.Exit(this, $"{nameof(RetryStrategyAdapter)}.{nameof(ShouldRetry)}");
}
}
}
/// <summary>
/// Runs a given action according to a retry policy with return value.
/// </summary>
/// <typeparam name="T">The action's return type</typeparam>
/// <param name="action">The action to run</param>
/// <param name="retryPolicy">The retry policy</param>
/// <returns></returns>
public static T Do <T>(
Func <T> action,
IRetryPolicy retryPolicy)
{
var exceptions = new List <Exception>();
int retryCount = 0;
Exception lastException = null;
TimeSpan retryInterval = TimeSpan.FromSeconds(0);
while (retryPolicy.ShouldRetry(retryCount, lastException, out retryInterval))
{
try
{
Thread.Sleep(retryInterval);
return(action());
}
catch (Exception ex)
{
lastException = ex;
exceptions.Add(ex);
}
}
throw new AggregateException(exceptions);
}
private void VerifyBackoffTimeOverflow(IRetryPolicy retryPolicy, int maxAttempts)
{
StorageException e = new StorageException();
OperationContext context = new OperationContext();
TimeSpan retryInterval;
TimeSpan previousRetryInterval = TimeSpan.FromMilliseconds(1); // larger than zero to ensure we never get zero back
for (int i = 0; i < maxAttempts; i++)
{
Assert.IsTrue(retryPolicy.ShouldRetry(i, (int)HttpStatusCode.InternalServerError, e, out retryInterval, context), string.Format("Attempt: {0}", i));
Assert.IsTrue(retryInterval >= previousRetryInterval, string.Format("Retry Interval: {0}, Previous Retry Interval: {1}, Attempt: {2}", retryInterval, previousRetryInterval, i));
previousRetryInterval = retryInterval;
}
Assert.IsFalse(retryPolicy.ShouldRetry(maxAttempts, (int)HttpStatusCode.InternalServerError, e, out retryInterval, context));
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subFolder,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy,
Func <string> getSharedAccessSignature)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
{
}
Exception lastException = null;
CloudBlockBlob blob = null;
// stats from azurescope show 10 to be an optimal number of transfer threads
int numThreads = ParallelTransferThreadCount;
var file = new FileInfo(localFile);
long fileSize = file.Length;
int maxBlockSize = GetBlockSize(fileSize);
// Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
// the 'key' is block id and 'value' is the block length.
List <string> blockList;
var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
int exceptionCount = 0;
blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
blob.DeleteIfExists(options: new BlobRequestOptions()
{
RetryPolicy = retryPolicy
});
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
var options = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
// Launch threads to upload blocks.
var tasks = new List <Task>();
long bytesSent = 0;
Action action = () =>
{
List <Exception> exceptions = new List <Exception>();
int sasRetry = 0;
if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
{
Thread.Sleep(_forceSharedAccessSignatureRetry);
}
if (queue.Count > 0)
{
FileStream fs = null;
try
{
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
KeyValuePair <int, int> blockIdAndLength;
while (queue.TryDequeue(out blockIdAndLength))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var buffer = new byte[blockIdAndLength.Value];
var binaryReader = new BinaryReader(fs);
// move the file system reader to the proper position
fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);
if (fileEncryption != null)
{
lock (fileEncryption)
{
using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
{
encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
}
}
}
using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
{
string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
string blockHash = GetMd5HashFromStream(buffer);
if (blob != null)
{
blob.PutBlock(blockIdString, ms, blockHash, options: options);
}
}
Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
var progress = (int)((double)bytesSent / file.Length * 100);
var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
OnTaskProgressChanged(eArgs);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
{
sasRetry++;
if (sasRetry > MaxSasSignatureRetry)
{
throw;
}
blob = GetCloudBlockBlob(uri, client, subFolder, localFile,
contentType, getSharedAccessSignature);
}
else
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
throw lastException;
}
Thread.Sleep(tm);
}
queue.Enqueue(blockIdAndLength);
}
catch (IOException ex)
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
throw lastException;
}
// dispose existing file stream
if (fs != null)
{
fs.Close();
}
Thread.Sleep(tm);
// try to reopen the file stream again
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
queue.Enqueue(blockIdAndLength);
}
}
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
};
for (int idxThread = 0; idxThread < numThreads; idxThread++)
{
tasks.Add(Task.Factory.StartNew(
action,
cancellationToken,
TaskCreationOptions.AttachedToParent,
TaskScheduler.Current));
}
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
{
if (result.Any(t => t.IsFaulted))
{
return;
}
try
{
blob.PutBlockList(blockList, options: options);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
{
blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
blob.PutBlockList(blockList, options: options);
}
else
{
throw;
}
}
}, TaskContinuationOptions.None).Wait(cancellationToken);
TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
}
private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature)
{
int exceptionCount = 0;
int numThreads = ParallelTransferThreadCount;
Exception lastException = null;
long bytesDownloaded = 0;
CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);
long blobLength = blob.Properties.Length;
int bufferLength = GetBlockSize(blobLength);
var queue = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads);
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
{
var tasks = new List <Task>();
Action action = () =>
{
KeyValuePair <long, int> blockOffsetAndLength;
int exceptionPerThread = 0;
int sasRetry = 0;
// A buffer to fill per read request.
var buffer = new byte[bufferLength];
if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
{
// The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
Thread.Sleep(_forceSharedAccessSignatureRetry);
}
while (queue.TryDequeue(out blockOffsetAndLength))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob);
using (var response = blobGetRequest.GetResponse() as HttpWebResponse)
{
if (response != null)
{
ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded);
var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100);
// raise the progress changed event
var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null);
OnTaskProgressChanged(eArgs);
}
}
}
catch (Exception ex)
{
var webEx = ex as WebException;
bool ok = (webEx != null) || ex is ObjectDisposedException;
bool isSasException = false;
if (!ok)
{
throw;
}
if (webEx != null && getSharedAccessSignature != null)
{
if (webEx.Response is HttpWebResponse)
{
var httpex = (HttpWebResponse)webEx.Response;
if (httpex.StatusCode == HttpStatusCode.Forbidden)
{
sasRetry++;
if (sasRetry > MaxSasSignatureRetry)
{
throw;
}
isSasException = true;
}
}
}
if (isSasException)
{
blob = InitializeCloudBlockBlob(uri, client, retryPolicy,
getSharedAccessSignature);
}
else
{
TimeSpan tm;
exceptionCount++;
exceptionPerThread++;
if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptionCount), ex);
break;
}
Thread.Sleep(tm);
}
// Add block back to queue
queue.Enqueue(blockOffsetAndLength);
}
}
};
// Launch threads to download chunks.
for (int idxThread = 0; idxThread < numThreads; idxThread++)
{
tasks.Add(Task.Factory.StartNew(action));
}
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, lastException, BlobTransferType.Download, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
Task.WaitAll(tasks.ToArray(), cancellationToken);
TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Download, localFile, uri);
}
}
private void UploadFileToBlob(
CancellationToken cancellationToken,
Uri uri,
string localFile,
string contentType,
string subFolder,
FileEncryption fileEncryption,
CloudBlobClient client,
IRetryPolicy retryPolicy)
{
//attempt to open the file first so that we throw an exception before getting into the async work
using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
{
}
Exception lastException = null;
CloudBlockBlob blob = null;
// stats from azurescope show 10 to be an optimal number of transfer threads
int numThreads = ParallelTransferThreadCount;
var file = new FileInfo(localFile);
long fileSize = file.Length;
int maxBlockSize = GetBlockSize(fileSize);
// Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
// the 'key' is block id and 'value' is the block length.
List<string> blockList;
var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
int exceptionCount = 0;
blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType);
blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy });
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
var options = new BlobRequestOptions
{
RetryPolicy = retryPolicy,
ServerTimeout = TimeSpan.FromSeconds(90)
};
// Launch threads to upload blocks.
var tasks = new List<Task>();
long bytesSent = 0;
Action action = () =>
{
List<Exception> exceptions = new List<Exception>();
if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
{
Thread.Sleep(_forceSharedAccessSignatureRetry);
}
if (queue.Count > 0)
{
FileStream fs = null;
try
{
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
KeyValuePair<int, int> blockIdAndLength;
while (queue.TryDequeue(out blockIdAndLength))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var buffer = new byte[blockIdAndLength.Value];
var binaryReader = new BinaryReader(fs);
// move the file system reader to the proper position
fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);
if (fileEncryption != null)
{
lock (fileEncryption)
{
using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
{
encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
}
}
}
using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
{
string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
string blockHash = GetMd5HashFromStream(buffer);
if (blob != null) blob.PutBlock(blockIdString, ms, blockHash, options: options);
}
Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
var progress = (int)((double)bytesSent / file.Length * 100);
var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
OnTaskProgressChanged(eArgs);
}
catch (StorageException ex)
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
throw lastException;
}
Thread.Sleep(tm);
queue.Enqueue(blockIdAndLength);
}
catch (IOException ex)
{
TimeSpan tm;
exceptionCount++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
{
lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
throw lastException;
}
// dispose existing file stream
if (fs != null)
{
fs.Close();
}
Thread.Sleep(tm);
// try to reopen the file stream again
fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
queue.Enqueue(blockIdAndLength);
}
}
}
finally
{
if (fs != null)
{
fs.Close();
}
}
}
};
for (int idxThread = 0; idxThread < numThreads; idxThread++)
{
tasks.Add(Task.Factory.StartNew(
action,
cancellationToken,
TaskCreationOptions.AttachedToParent,
TaskScheduler.Current));
}
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
{
if (result.Any(t => t.IsFaulted))
{
return;
}
blob.PutBlockList(blockList, options: options);
}, TaskContinuationOptions.None).Wait(cancellationToken);
TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
}
private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
{
int numThreads = ParallelTransferThreadCount;
List<Exception> exceptions = new List<Exception>();
AggregateException aggregateException = null;
long bytesDownloaded = 0;
CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy);
long blobLength = blob.Properties.Length;
int bufferLength = GetBlockSize(blobLength);
var queue = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads);
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
{
var tasks = new List<Task>();
Action action = () =>
{
KeyValuePair<long, int> blockOffsetAndLength;
int exceptionPerThread = 0;
// A buffer to fill per read request.
var buffer = new byte[bufferLength];
if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
{
// The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
Thread.Sleep(_forceSharedAccessSignatureRetry);
}
while (queue.TryDequeue(out blockOffsetAndLength))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob);
using (var response = blobGetRequest.GetResponse() as HttpWebResponse)
{
if (response != null)
{
ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded);
var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100);
// raise the progress changed event
var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null);
OnTaskProgressChanged(eArgs);
}
}
}
catch (Exception ex)
{
var webEx = ex as WebException;
bool ok = (webEx != null) || ex is ObjectDisposedException;
if (!ok)
{
throw;
}
if (webEx != null)
{
if (webEx.Response is HttpWebResponse)
{
var httpex = (HttpWebResponse)webEx.Response;
if (httpex.StatusCode == HttpStatusCode.Forbidden)
{
blob = InitializeCloudBlockBlob(uri, null, retryPolicy);
}
}
}
TimeSpan tm;
exceptionPerThread++;
exceptions.Add(ex);
if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext()))
{
aggregateException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptions.Count), exceptions);
throw aggregateException;
}
Thread.Sleep(tm);
// Add block back to queue
queue.Enqueue(blockOffsetAndLength);
}
}
};
// Launch threads to download chunks.
for (int idxThread = 0; idxThread < numThreads; idxThread++)
{
tasks.Add(Task.Factory.StartNew(action));
}
if (cancellationToken.IsCancellationRequested)
{
TaskCompletedCallback(true, aggregateException, BlobTransferType.Download, localFile, uri);
cancellationToken.ThrowIfCancellationRequested();
}
Task.WaitAll(tasks.ToArray(), cancellationToken);
TaskCompletedCallback(cancellationToken.IsCancellationRequested, aggregateException, BlobTransferType.Download, localFile, uri);
}
}
private static CloudBlockBlob InitializeCloudBlockBlob(Uri uri, CloudBlobClient client, IRetryPolicy retryPolicy)
{
CloudBlockBlob blob = null;
if (client != null)
{
blob = new CloudBlockBlob(uri, client.Credentials);
}
else
{
blob = new CloudBlockBlob(uri);
}
bool fetch = false;
bool shouldRetry = true;
TimeSpan delay;
int retryCount = 0;
StorageException lastException = null;
while (!fetch && shouldRetry)
{
try
{
blob.FetchAttributes(options: new BlobRequestOptions() { RetryPolicy = retryPolicy });
fetch = true;
}
catch (StorageException ex)
{
retryCount++;
lastException = ex;
shouldRetry = retryPolicy.ShouldRetry(retryCount, ex.RequestInformation.HttpStatusCode, lastException, out delay, new OperationContext());
Thread.Sleep(delay);
}
}
if (!fetch)
{
throw lastException;
}
return blob;
}
private static async Task <TResult> ExecuteUnderRetryPolicy <TResult>(Func <Task <TResult> > executionMethod, CancellationToken cancellationToken, OperationContext operationContext, TableRequestOptions requestOptions)
{
IRetryPolicy retryPolicyInstance = null;
int retryCount = 0;
DateTime startTime = DateTime.UtcNow;
StorageException previousException = null;
while (true)
{
operationContext = (operationContext ?? new OperationContext());
TimeSpan retryInterval;
try
{
ThrowTimeoutIfElapsed(requestOptions.MaximumExecutionTime, startTime, operationContext, previousException);
ThrowCancellationIfRequested(cancellationToken);
return(await executionMethod());
}
// catch (NotFoundException ex)
// {
// Logger.LogError(operationContext, "Retry policy did not allow for a retry. Failing with {0}.", ex.Message);
// throw;
// }
catch (StorageException ex2)
{
if (retryPolicyInstance == null)
{
retryPolicyInstance = requestOptions.RetryPolicy.CreateInstance();
}
if (!ex2.IsRetryable)
{
Logger.LogError(operationContext, "Retry policy did not allow for a retry. Failing with {0}.", ex2.Message);
throw;
}
int statusCode = (ex2.RequestInformation != null) ? ex2.RequestInformation.HttpStatusCode : 500;
if (!retryPolicyInstance.ShouldRetry(retryCount++, statusCode, ex2, out retryInterval, operationContext))
{
Logger.LogError(operationContext, "Retry policy did not allow for a retry. Failing with {0}.", ex2.Message);
throw;
}
previousException = ex2;
}
catch (Exception ex)
{
if (ex.GetType().Name == "NotFoundException")
{
Logger.LogError(operationContext, "Retry policy did not allow for a retry. Failing with {0}.", ex.Message);
throw;
}
}
if (retryInterval < TimeSpan.Zero || retryInterval > MaximumRetryBackoff)
{
retryInterval = MaximumRetryBackoff;
}
if (retryInterval != TimeSpan.Zero)
{
ThrowTimeoutIfElapsed(requestOptions.MaximumExecutionTime, startTime, operationContext, previousException);
Logger.LogInformational(operationContext, "Operation will be retried after {0}ms.", (int)retryInterval.TotalMilliseconds);
await Task.Delay(retryInterval);
}
operationContext?.FireRetrying(new RequestEventArgs(operationContext.LastResult));
}
}
static void RetryTask(IRetryPolicy retryPolicy, int retryCount, CancellationToken cancellationToken,
Func <Task> action, Action onSuccess = null, Action <Exception> onFinalError = null, Action onCancel = null)
{
Task mainTask;
try
{
mainTask = action();
}
catch (Exception exception)
{
if (onFinalError != null)
{
onFinalError(exception);
}
else
{
throw;
}
return;
}
mainTask.ContinueWith(task =>
{
try
{
bool isCancellationRequested = cancellationToken.IsCancellationRequested;
if (task.IsFaulted)
{
var baseException = task.Exception.GetBaseException();
if (isCancellationRequested || baseException is TaskCanceledException)
{
if (onCancel != null)
{
onCancel();
}
return;
}
// Fail task if we don't retry
TimeSpan retryDelay;
if (retryPolicy == null || !retryPolicy.ShouldRetry(retryCount, 0, baseException, out retryDelay, null))
{
if (onFinalError != null)
{
onFinalError(baseException);
}
return;
}
// Retry immediately
if (retryDelay <= TimeSpan.Zero)
{
RetryTask(retryPolicy, retryCount + 1, cancellationToken, action, onSuccess, onFinalError, onCancel);
return;
}
new Timer(self =>
{
// Consider to use TaskEx.Delay instead once available
((IDisposable)self).Dispose();
// Do not retry oif cancelled in the meantime
if (cancellationToken.IsCancellationRequested)
{
if (onCancel != null)
{
onCancel();
}
return;
}
RetryTask(retryPolicy, retryCount + 1, cancellationToken, action, onSuccess, onFinalError, onCancel);
}).Change(retryDelay, TimeSpan.FromMilliseconds(-1));
return;
}
if (task.IsCanceled || isCancellationRequested)
{
if (onCancel != null)
{
onCancel();
}
return;
}
if (onSuccess != null)
{
onSuccess();
}
}
catch (Exception e)
{
// we do not retry if the exception happened in the continuation
if (onFinalError != null)
{
onFinalError(e);
}
}
});
}
public bool ShouldRetry(int currentRetryCount, int statusCode, Exception lastException, out TimeSpan retryInterval,
OperationContext operationContext)
{
return(_retryPolicy.ShouldRetry(currentRetryCount, statusCode, lastException, out retryInterval, operationContext));
}