/// <summary>
/// Gets the retry decision based on the request error
/// </summary>
internal static RetryDecisionWithReason GetRetryDecisionWithReason(
IRequestError error, IExtendedRetryPolicy policy, IStatement statement, Configuration config, int retryCount)
{
var ex = error.Exception;
if (ex is SocketException || ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException ||
ex is OperationTimedOutException)
{
if (ex is SocketException exception)
{
RequestExecution.Logger.Verbose("Socket error " + exception.SocketErrorCode);
}
// For PREPARE requests, retry on next host
var decision = statement == null && ex is OperationTimedOutException
? RetryDecision.Retry(null, false)
: policy.OnRequestError(statement, config, ex, retryCount);
return(new RetryDecisionWithReason(decision, RequestExecution.GetErrorType(error)));
}
if (ex is ReadTimeoutException e)
{
return(new RetryDecisionWithReason(
policy.OnReadTimeout(
statement,
e.ConsistencyLevel,
e.RequiredAcknowledgements,
e.ReceivedAcknowledgements,
e.WasDataRetrieved,
retryCount),
RequestErrorType.ReadTimeOut
));
}
if (ex is WriteTimeoutException e1)
{
return(new RetryDecisionWithReason(
policy.OnWriteTimeout(
statement,
e1.ConsistencyLevel,
e1.WriteType,
e1.RequiredAcknowledgements,
e1.ReceivedAcknowledgements,
retryCount),
RequestErrorType.WriteTimeOut
));
}
if (ex is UnavailableException e2)
{
return(new RetryDecisionWithReason(
policy.OnUnavailable(statement, e2.Consistency, e2.RequiredReplicas, e2.AliveReplicas, retryCount),
RequestErrorType.Unavailable
));
}
// Any other Exception just throw it
return(new RetryDecisionWithReason(RetryDecision.Rethrow(), RequestExecution.GetErrorType(error)));
}
/// <summary>
/// Creates a new <see cref="IExtendedRetryPolicy"/> that logs the decision of the provided <c>policy</c>.
/// </summary>
/// <param name="policy"> the policy to wrap. The policy created by this
/// constructor will return the same decision than <c>policy</c> but will log them.</param>
public LoggingRetryPolicy(IRetryPolicy policy)
{
_policy = policy;
// Use the provided policy for extended policy methods.
// If the provided policy is not IExtendedRetryPolicy, use the default.
_extendedPolicy = (policy as IExtendedRetryPolicy) ?? new DefaultRetryPolicy();
}
/// <summary>
/// Creates a new instance of <see cref="IdempotenceAwareRetryPolicy"/>.
/// </summary>
/// <param name="childPolicy">The retry policy to wrap.</param>
public IdempotenceAwareRetryPolicy(IRetryPolicy childPolicy)
{
if (childPolicy == null)
{
throw new ArgumentNullException("childPolicy");
}
_childPolicy = childPolicy;
_extendedChildPolicy = childPolicy as IExtendedRetryPolicy;
}
public MultiLocationTestHelper(StorageUri storageUri, StorageLocation initialLocation, IList <RetryContext> retryContextList, IList <RetryInfo> retryInfoList)
{
this.storageUri = storageUri;
this.initialLocation = initialLocation;
this.retryContextList = retryContextList;
this.retryInfoList = retryInfoList;
this.OperationContext = new OperationContext();
this.OperationContext.SendingRequest += this.SendingRequest;
this.RetryPolicy = new AlwaysRetry(this.retryContextList, this.retryInfoList);
}
/// <summary>
/// When the policy provided implements IExtendedRetryPolicy, it returns the same instance.
/// Otherwise it returns a new instance of IExtendedRetryPolicy, delegating all <see cref="IRetryPolicy"/>
/// methods to the provided policy and the rest to the defaultPolicy.
/// </summary>
/// <param name="policy">The policy to wrap or cast</param>
/// <param name="defaultPolicy">
/// The default policy to handle IExtendedRetryPolicy methods.
/// When null, the default retry policy will be used.
/// </param>
internal static IExtendedRetryPolicy Wrap(this IRetryPolicy policy, IExtendedRetryPolicy defaultPolicy)
{
var resultPolicy = policy as IExtendedRetryPolicy;
if (resultPolicy == null)
{
// Wrap the user provided policy
return(new WrappedExtendedRetryPolicy(policy, defaultPolicy ?? new DefaultRetryPolicy()));
}
// Return the user-provided policy casted to IExtendedRetryPolicy
return(resultPolicy);
}
internal Policies(ILoadBalancingPolicy loadBalancingPolicy,
IReconnectionPolicy reconnectionPolicy,
IRetryPolicy retryPolicy,
ISpeculativeExecutionPolicy speculativeExecutionPolicy,
ITimestampGenerator timestampGenerator)
{
_loadBalancingPolicy = loadBalancingPolicy ?? DefaultLoadBalancingPolicy;
_reconnectionPolicy = reconnectionPolicy ?? DefaultReconnectionPolicy;
_retryPolicy = retryPolicy ?? DefaultRetryPolicy;
_speculativeExecutionPolicy = speculativeExecutionPolicy ?? DefaultSpeculativeExecutionPolicy;
_timestampGenerator = timestampGenerator ?? DefaultTimestampGenerator;
_extendedRetryPolicy = _retryPolicy.Wrap(DefaultExtendedRetryPolicy);
}
private Policies(
ILoadBalancingPolicy loadBalancingPolicy,
IReconnectionPolicy reconnectionPolicy,
IRetryPolicy retryPolicy,
ISpeculativeExecutionPolicy speculativeExecutionPolicy,
ITimestampGenerator timestampGenerator,
IExtendedRetryPolicy extendedRetryPolicy)
{
_loadBalancingPolicy = loadBalancingPolicy ?? throw new ArgumentNullException(nameof(loadBalancingPolicy));
_reconnectionPolicy = reconnectionPolicy ?? throw new ArgumentNullException(nameof(reconnectionPolicy));
_retryPolicy = retryPolicy ?? throw new ArgumentNullException(nameof(retryPolicy));
_speculativeExecutionPolicy = speculativeExecutionPolicy ?? throw new ArgumentNullException(nameof(speculativeExecutionPolicy));
_timestampGenerator = timestampGenerator ?? throw new ArgumentNullException(nameof(timestampGenerator));
_extendedRetryPolicy = extendedRetryPolicy ?? throw new ArgumentNullException(nameof(extendedRetryPolicy));
}
private void Initialize(
ConsistencyLevel?consistencyLevel,
ConsistencyLevel?serialConsistencyLevel,
int?readTimeoutMillis,
ILoadBalancingPolicy loadBalancingPolicy,
ISpeculativeExecutionPolicy speculativeExecutionPolicy,
IExtendedRetryPolicy retryPolicy)
{
ConsistencyLevel = consistencyLevel;
SerialConsistencyLevel = serialConsistencyLevel;
ReadTimeoutMillis = readTimeoutMillis;
LoadBalancingPolicy = loadBalancingPolicy;
SpeculativeExecutionPolicy = speculativeExecutionPolicy;
RetryPolicy = retryPolicy;
}
internal ExecutionProfile(
ConsistencyLevel?consistencyLevel,
ConsistencyLevel?serialConsistencyLevel,
int?readTimeoutMillis,
ILoadBalancingPolicy loadBalancingPolicy,
ISpeculativeExecutionPolicy speculativeExecutionPolicy,
IExtendedRetryPolicy retryPolicy)
{
Initialize(
consistencyLevel,
serialConsistencyLevel,
readTimeoutMillis,
loadBalancingPolicy,
speculativeExecutionPolicy,
retryPolicy);
}
/// <summary>
/// Gets the retry decision based on the exception from Cassandra
/// </summary>
public static RetryDecision GetRetryDecision(Exception ex, IExtendedRetryPolicy policy, IStatement statement,
Configuration config, int retryCount)
{
if (ex is SocketException)
{
Logger.Verbose("Socket error " + ((SocketException)ex).SocketErrorCode);
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException)
{
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is ReadTimeoutException)
{
var e = (ReadTimeoutException)ex;
return(policy.OnReadTimeout(statement, e.ConsistencyLevel, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, e.WasDataRetrieved, retryCount));
}
if (ex is WriteTimeoutException)
{
var e = (WriteTimeoutException)ex;
return(policy.OnWriteTimeout(statement, e.ConsistencyLevel, e.WriteType, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, retryCount));
}
if (ex is UnavailableException)
{
var e = (UnavailableException)ex;
return(policy.OnUnavailable(statement, e.Consistency, e.RequiredReplicas, e.AliveReplicas, retryCount));
}
if (ex is OperationTimedOutException)
{
if (statement == null)
{
// For PREPARE requests, retry on next host
return(RetryDecision.Retry(null, false));
}
// Delegate on retry policy
return(policy.OnRequestError(statement, config, ex, retryCount));
}
// Any other Exception just throw it
return(RetryDecision.Rethrow());
}
private static void VerifyRetryInfoList(IExtendedRetryPolicy retryPolicy, HttpStatusCode primaryStatusCode, HttpStatusCode secondaryStatusCode, LocationMode locationMode, Func <int, double> allowedDelta, params RetryInfo[] expectedRetryInfoList)
{
StorageLocation initialLocation = GetInitialLocation(locationMode);
StorageLocation currentLocation = GetNextLocation(locationMode, initialLocation);
OperationContext operationContext = new OperationContext();
RequestResult requestResult = new RequestResult()
{
Exception = new Exception(),
TargetLocation = initialLocation,
HttpStatusCode = initialLocation == StorageLocation.Primary ? (int)primaryStatusCode : (int)secondaryStatusCode,
StartTime = DateTime.Now,
EndTime = DateTime.Now,
};
for (int i = 0; i < expectedRetryInfoList.Length; i++)
{
RetryInfo retryInfo = retryPolicy.Evaluate(new RetryContext(i, requestResult, currentLocation, locationMode), operationContext);
string message = string.Format("Failed at retry {0}", i);
Assert.IsNotNull(retryInfo, message);
Assert.AreEqual(expectedRetryInfoList[i].TargetLocation, retryInfo.TargetLocation, message);
Assert.AreEqual(expectedRetryInfoList[i].UpdatedLocationMode, retryInfo.UpdatedLocationMode, message);
Assert.AreEqual(expectedRetryInfoList[i].RetryInterval.TotalSeconds, retryInfo.RetryInterval.TotalSeconds, allowedDelta(i), message);
Thread.Sleep(retryInfo.RetryInterval);
requestResult.TargetLocation = retryInfo.TargetLocation;
requestResult.HttpStatusCode = retryInfo.TargetLocation == StorageLocation.Primary ? (int)primaryStatusCode : (int)secondaryStatusCode;
requestResult.StartTime = DateTime.Now;
requestResult.EndTime = DateTime.Now;
locationMode = retryInfo.UpdatedLocationMode;
currentLocation = GetNextLocation(locationMode, currentLocation);
}
Assert.IsNull(retryPolicy.Evaluate(new RetryContext(expectedRetryInfoList.Length, requestResult, currentLocation, locationMode), operationContext));
}
internal static RetryDecision GetRetryDecisionFromClientError(Exception ex, IExtendedRetryPolicy policy, IStatement statement, Configuration config, int retryCount)
{
return(RequestExecution.GetRetryDecisionWithReason(RequestError.CreateClientError(ex, false), policy, statement, config, retryCount).Decision);
}
public static T ExecuteSync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// Steps 1-4
Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestSync);
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made.
// Therefore, we will not get extended error info. Hence ParseError doesn't matter here.
StorageException storageEx = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// 5. potentially upload data
if (cmd.SendStream != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginGetRequestStream;
Logger.LogInformational(executionState.OperationContext, SR.TracePrepareUpload);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.ReqStream = executionState.Req.GetRequestStream();
executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest;
Logger.LogInformational(executionState.OperationContext, SR.TraceUpload);
MultiBufferMemoryStream multiBufferMemoryStream = cmd.SendStream as MultiBufferMemoryStream;
try
{
if (multiBufferMemoryStream != null && !cmd.SendStreamLength.HasValue)
{
multiBufferMemoryStream.FastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime);
}
else
{
// don't calculate md5 here as we should have already set this for auth purposes
executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, cmd.SendStreamLength, null /* maxLength */, false, true, executionState, null /* streamCopyState */);
}
executionState.ReqStream.Flush();
executionState.ReqStream.Dispose();
executionState.ReqStream = null;
}
catch (Exception)
{
executionState.Req.Abort();
throw;
}
}
// 6. Get response
try
{
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.Resp = (HttpWebResponse)executionState.Req.GetResponse();
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
}
catch (WebException ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
executionState.Resp = (HttpWebResponse)ex.Response;
if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
{
throw new TimeoutException();
}
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
}
}
// Response
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = executionState.Resp.GetResponseStream();
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
cmd.ResponseStream.WriteToSync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, false, executionState, cmd.StreamCopyState);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
StorageException translatedException = ExecutorBase.TranslateExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
cmd.CurrentResult,
executionState.CurrentLocation,
cmd.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
Thread.Sleep(delay);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
Executor.FireRetrying(executionState);
}while (shouldRetry);
}
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
private static void EndOperation <T>(ExecutionState <T> executionState)
{
Executor.FinishRequestAttempt(executionState);
try
{
// If an operation has been canceled of timed out this should overwrite any exception
Executor.CheckCancellation(executionState);
Executor.CheckTimeout(executionState, true);
// Success
if (executionState.ExceptionRef == null)
{
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState);
executionState.OnComplete();
return;
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TracePostProcessError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
}
finally
{
try
{
if (executionState.ReqStream != null)
{
executionState.ReqStream.Dispose();
executionState.ReqStream = null;
}
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
catch (Exception)
{
// no op
}
}
// Handle Retry
try
{
StorageException translatedException = TranslateExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, executionState.RestCMD.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
executionState.Cmd.CurrentResult,
executionState.CurrentLocation,
executionState.RestCMD.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
executionState.RestCMD.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
executionState.UpdateCompletedSynchronously(false);
if (executionState.BackoffTimer == null)
{
executionState.BackoffTimer = new Timer(
Executor.RetryRequest <T>,
executionState,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
}
executionState.CancelDelegate = () =>
{
// Disabling the timer here, but there is still a scenario where the user calls cancel after
// the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
// if that happens, next retry will start and then stop immediately because of the cancelled flag.
Timer backoffTimer = executionState.BackoffTimer;
if (backoffTimer != null)
{
executionState.BackoffTimer = null;
backoffTimer.Dispose();
}
Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
Executor.CheckCancellation(executionState);
executionState.OnComplete();
};
}
else
{
// Start next request immediately
Executor.RetryRequest <T>(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.OnComplete();
}
}
public static async Task <T> ExecuteAsync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
// Create a new client
HttpClient client = cmd.HttpClient ?? HttpClientFactory.Instance;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// Steps 1-4: Build Content/SetHeaders/
Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestAsync, timeoutTokenSource);
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made.
// Therefore, we will not get extended error info. Hence ParseError doesn't matter here.
StorageException storageEx = await ExecutorBase.TranslateExceptionBasedOnParseErrorAsync(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, token).ConfigureAwait(false);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// Send Request
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutTokenSource.Token).ConfigureAwait(false);
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
// Check that echoed client ID matches the one we sent
var clientRequestId = HttpRequestParsers.GetHeader(executionState.Req, Constants.HeaderConstants.ClientRequestIdHeader);
var echoedClientRequestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.ClientRequestIdHeader);
if (echoedClientRequestId != null && echoedClientRequestId != clientRequestId)
{
var requestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.RequestIdHeader);
var storageEx = new StorageException($"Echoed client request ID: {echoedClientRequestId} does not match sent client request ID: {clientRequestId}. Service request ID: {requestId}")
{
IsRetryable = false
};
executionState.ExceptionRef = storageEx;
throw storageEx;
}
// Since HttpClient wont throw for non success, manually check and populate an exception
if (!executionState.Resp.IsSuccessStatusCode)
{
// At this point, don't try to read the stream to parse the error
executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, token, executionState.Cmd.ParseErrorAsync).ConfigureAwait(false);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.ContentCrc64, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
try
{
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception ex)
{
executionState.ExceptionRef = ex;
}
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
var responseStream = await executionState.Resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
if (cmd.NetworkTimeout.HasValue)
{
responseStream = new TimeoutStream(responseStream, cmd.NetworkTimeout.Value);
}
cmd.ResponseStream = responseStream;
// The stream is now available in ResponseStream. Use the stream to parse out the response or error
if (executionState.ExceptionRef != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);
try
{
cmd.ErrorStream = new MemoryStream();
await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, ChecksumRequested.None, executionState, new StreamDescriptor(), timeoutTokenSource.Token).ConfigureAwait(false);
cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream, executionState.Resp);
throw executionState.ExceptionRef;
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
cmd.ErrorStream.Dispose();
cmd.ErrorStream = null;
}
}
else
{
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, cmd.ChecksumRequestedForResponseStream, executionState, cmd.StreamCopyState, timeoutTokenSource.Token).ConfigureAwait(false);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
}
await Executor.ProcessEndOfRequestAsync(executionState, token).ConfigureAwait(false);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
if (e is OperationCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
e = new TimeoutException(SR.TimeoutExceptionMessage, e);
}
StorageException translatedException = await ExecutorBase.TranslateExceptionBasedOnParseErrorAsync(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, token).ConfigureAwait(false);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
cmd.CurrentResult,
executionState.CurrentLocation,
cmd.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Dispose();
executionState.Resp = null;
}
}
// potentially backoff
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
// I.E. Rewind stream etc.
cmd.RecoveryAction?.Invoke(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay, token).ConfigureAwait(false);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
Executor.FireRetrying(executionState);
}while (shouldRetry);
// should never get here
throw new NotImplementedException(SR.InternalStorageError);
}
}
public IExecutionProfileBuilder WithRetryPolicy(IExtendedRetryPolicy retryPolicy)
{
_retryPolicy = retryPolicy ?? throw new ArgumentNullException(nameof(retryPolicy));
return(this);
}
public MultiLocationTestHelper(StorageUri storageUri, StorageLocation initialLocation, IList<RetryContext> retryContextList, IList<RetryInfo> retryInfoList)
{
this.storageUri = storageUri;
this.initialLocation = initialLocation;
this.retryContextList = retryContextList;
this.retryInfoList = retryInfoList;
this.OperationContext = new OperationContext();
this.OperationContext.SendingRequest += this.SendingRequest;
this.RetryPolicy = new AlwaysRetry(this.retryContextList, this.retryInfoList);
}
public WrappedExtendedRetryPolicy(IRetryPolicy policy, IExtendedRetryPolicy defaultPolicy)
{
Policy = policy;
DefaultPolicy = defaultPolicy;
}
public WrappedExtendedRetryPolicy(IRetryPolicy policy, IExtendedRetryPolicy defaultPolicy)
{
Policy = policy ?? throw new ArgumentNullException(nameof(policy));
DefaultPolicy = defaultPolicy ?? throw new ArgumentNullException(nameof(defaultPolicy));
}
public static async Task <T> ExecuteAsync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
{
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
{
using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
{
CancellationToken timeoutToken = timeoutTokenSource.Token;
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
HttpClient client = cmd.HttpClient ?? HttpClientFactory.Instance;
do
{
try
{
executionState.Init();
ExecutionStateUtils.StartRequestAttempt(executionState);
ProcessStartOfRequest(executionState, "Starting asynchronous request to {0}.", timeoutTokenSource);
ExecutionStateUtils.CheckTimeout(executionState, throwOnTimeout: true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, "Exception thrown while initializing request: {0}.", ex.Message);
StorageException ex2 = await ExecutionStateUtils.TranslateExceptionBasedOnParseErrorAsync(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, timeoutToken).ConfigureAwait(continueOnCapturedContext: false);
ex2.IsRetryable = false;
executionState.ExceptionRef = ex2;
throw executionState.ExceptionRef;
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, "Waiting for response.");
ExecutionState <T> executionState2 = executionState;
executionState2.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutToken).ConfigureAwait(continueOnCapturedContext: false);
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
if (!executionState.Resp.IsSuccessStatusCode)
{
executionState2 = executionState;
executionState2.ExceptionRef = await StorageExceptionTranslator.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, timeoutToken, executionState.Cmd.ParseErrorAsync).ConfigureAwait(continueOnCapturedContext: false);
}
Logger.LogInformational(executionState.OperationContext, "Response received. Status code = {0}, Request ID = {1}, Content-MD5 = {2}, ETag = {3}.", executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
ExecutionStateUtils.FireResponseReceived(executionState);
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
try
{
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
executionState.ExceptionRef = null;
}
catch (Exception exceptionRef)
{
executionState.ExceptionRef = exceptionRef;
}
Logger.LogInformational(executionState.OperationContext, "Response headers were processed successfully, proceeding with the rest of the operation.");
}
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = new MemoryStream();
await(await executionState.Resp.Content.ReadAsStreamAsync()).WriteToAsync(cmd.ResponseStream, executionState, timeoutTokenSource.Token);
cmd.ResponseStream.Position = 0L;
if (executionState.ExceptionRef != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, "Downloading error response body.");
try
{
executionState2 = executionState;
executionState2.ExceptionRef = await StorageExceptionTranslator.TranslateExceptionWithPreBufferedStreamAsync(executionState.ExceptionRef, executionState.Cmd.CurrentResult, cmd.ResponseStream, executionState.Resp, executionState.Cmd.ParseErrorAsync, timeoutToken);
throw executionState.ExceptionRef;
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
await ProcessEndOfRequestAsync(executionState, timeoutToken).ConfigureAwait(continueOnCapturedContext: false);
ExecutionStateUtils.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception ex3)
{
Exception ex4 = ex3;
Logger.LogWarning(executionState.OperationContext, "Exception thrown during the operation: {0}.", ex4.Message);
ExecutionStateUtils.FinishRequestAttempt(executionState);
if (ex4 is OperationCanceledException && executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0)
{
ex4 = new TimeoutException("The client could not finish the operation within specified timeout.", ex4);
}
Exception ex6 = executionState.ExceptionRef = await ExecutionStateUtils.TranslateExceptionBasedOnParseErrorAsync(ex4, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, timeoutToken).ConfigureAwait(continueOnCapturedContext: false);
Logger.LogInformational(executionState.OperationContext, "Checking if the operation should be retried. Retry count = {0}, HTTP status code = {1}, Retryable exception = {2}, Exception = {3}.", executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, ((StorageException)ex6).IsRetryable ? "yes" : "no", ex6.Message);
shouldRetry = false;
if (((StorageException)ex6).IsRetryable && executionState.RetryPolicy != null)
{
executionState.CurrentLocation = ExecutionStateUtils.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, "The next location has been set to {0}, based on the location mode.", executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(new RetryContext(executionState.RetryCount++, cmd.CurrentResult, executionState.CurrentLocation, cmd.LocationMode), executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, "The extended retry policy set the next location to {0} and updated the location mode to {1}.", retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(executionState.RetryCount++, cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext);
}
if (delay < TimeSpan.Zero || delay > TableRestConstants.MaximumRetryBackoff)
{
delay = TableRestConstants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Dispose();
executionState.Resp = null;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? "Operation cannot be retried because the maximum execution time has been reached. Failing with {0}." : "Retry policy did not allow for a retry. Failing with {0}.", executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
cmd.RecoveryAction?.Invoke(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay, timeoutToken).ConfigureAwait(continueOnCapturedContext: false);
}
Logger.LogInformational(executionState.OperationContext, "Retrying failed operation.");
ExecutionStateUtils.FireRetrying(executionState);
}while (shouldRetry);
throw new NotImplementedException("Unexpected internal storage client error.");
}
}
}
/// <summary>
/// Sets the current policy as extended retry policy.
/// If the current policy is not <see cref="IExtendedRetryPolicy"/>, it creates a wrapper to delegate
/// the methods that were not implemented to a default policy.
/// </summary>
internal void InitializeRetryPolicy(ICluster cluster)
{
_extendedRetryPolicy = _retryPolicy.Wrap(null);
}
private async static Task <T> ExecuteAsyncInternal <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
{
// Note all code below will reference state, not params directly, this will allow common code with multiple executors (APM, Sync, Async)
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
// Note - The service accepts both api-version and x-ms-version and therefore it is ok to add x-ms-version to all requests.
// Create a new client
HttpClient client = HttpClientFactory.Instance;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// 1. Build request and content
executionState.CurrentOperation = ExecutorOperation.BeginOperation;
// Content is re-created every retry, as HttpClient disposes it after a successful request
HttpContent content = cmd.BuildContent != null?cmd.BuildContent(cmd, executionState.OperationContext) : null;
// This is so the old auth header etc is cleared out, the content is where serialization occurs which is the major perf hit
Uri uri = cmd.StorageUri.GetUri(executionState.CurrentLocation);
Uri transformedUri = cmd.Credentials.TransformUri(uri);
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, transformedUri);
UriQueryBuilder builder = new UriQueryBuilder(executionState.RestCMD.Builder);
executionState.Req = cmd.BuildRequest(cmd, transformedUri, builder, content, cmd.ServerTimeoutInSeconds, executionState.OperationContext);
// 2. Set Headers
Executor.ApplyUserHeaders(executionState);
// Let the user know we are ready to send
Executor.FireSendingRequest(executionState);
// 3. Sign Request is not needed, as HttpClient will call us
// 4. Set timeout
if (executionState.OperationExpiryTime.HasValue)
{
// set the token to cancel after timing out, if the higher token hasn't already been cancelled
timeoutTokenSource.CancelAfter(executionState.RemainingTimeout);
}
else
{
// effectively prevent timeout
timeoutTokenSource.CancelAfter(int.MaxValue);
}
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default
StorageException storageEx = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult, null);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// Send Request
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutTokenSource.Token);
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
// Since HttpClient wont throw for non success, manually check and populate an exception
if (!executionState.Resp.IsSuccessStatusCode)
{
// At this point, don't try to read the stream to parse the error
executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, executionState.Cmd.ParseError);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
try
{
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception ex)
{
executionState.ExceptionRef = ex;
}
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = await executionState.Resp.Content.ReadAsStreamAsync();
// The stream is now available in ResponseStream. Use the stream to parse out the response or error
if (executionState.ExceptionRef != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);
try
{
cmd.ErrorStream = new MemoryStream();
await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, null /* copyLength */, null /* maxLength */, false, executionState, new StreamDescriptor(), timeoutTokenSource.Token);
cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
#if ASPNET_K || PORTABLE
executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream, executionState.Resp);
#else
executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream);
#endif
throw executionState.ExceptionRef;
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
cmd.ErrorStream.Dispose();
cmd.ErrorStream = null;
}
}
else
{
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, executionState, cmd.StreamCopyState, timeoutTokenSource.Token);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
}
// 9. Evaluate Response & Parse Results, (Stream potentially available here)
if (cmd.PostProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PostProcess;
Logger.LogInformational(executionState.OperationContext, SR.TracePostProcess);
executionState.Result = await cmd.PostProcessResponse(cmd, executionState.Resp, executionState.OperationContext);
}
executionState.CurrentOperation = ExecutorOperation.EndOperation;
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
if (e is TaskCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
e = new TimeoutException(SR.TimeoutExceptionMessage, e);
}
#if ASPNET_K || PORTABLE
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), executionState.Resp);
#else
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null));
#endif
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
cmd.CurrentResult,
executionState.CurrentLocation,
cmd.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Dispose();
executionState.Resp = null;
}
}
// potentially backoff
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
cmd.RecoveryAction(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay, token);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
Executor.FireRetrying(executionState);
}while (shouldRetry);
// should never get here
throw new NotImplementedException(SR.InternalStorageError);
}
}
private static void VerifyRetryInfoList(IExtendedRetryPolicy retryPolicy, HttpStatusCode primaryStatusCode, HttpStatusCode secondaryStatusCode, LocationMode locationMode, Func<int, double> allowedDelta, params RetryInfo[] expectedRetryInfoList)
{
StorageLocation initialLocation = GetInitialLocation(locationMode);
StorageLocation currentLocation = GetNextLocation(locationMode, initialLocation);
OperationContext operationContext = new OperationContext();
RequestResult requestResult = new RequestResult()
{
Exception = new Exception(),
TargetLocation = initialLocation,
HttpStatusCode = initialLocation == StorageLocation.Primary ? (int)primaryStatusCode : (int)secondaryStatusCode,
StartTime = DateTime.Now,
EndTime = DateTime.Now,
};
for (int i = 0; i < expectedRetryInfoList.Length; i++)
{
RetryInfo retryInfo = retryPolicy.Evaluate(new RetryContext(i, requestResult, currentLocation, locationMode), operationContext);
string message = string.Format("Failed at retry {0}", i);
Assert.IsNotNull(retryInfo, message);
Assert.AreEqual(expectedRetryInfoList[i].TargetLocation, retryInfo.TargetLocation, message);
Assert.AreEqual(expectedRetryInfoList[i].UpdatedLocationMode, retryInfo.UpdatedLocationMode, message);
Assert.AreEqual(expectedRetryInfoList[i].RetryInterval.TotalSeconds, retryInfo.RetryInterval.TotalSeconds, allowedDelta(i), message);
Thread.Sleep(retryInfo.RetryInterval);
requestResult.TargetLocation = retryInfo.TargetLocation;
requestResult.HttpStatusCode = retryInfo.TargetLocation == StorageLocation.Primary ? (int)primaryStatusCode : (int)secondaryStatusCode;
requestResult.StartTime = DateTime.Now;
requestResult.EndTime = DateTime.Now;
locationMode = retryInfo.UpdatedLocationMode;
currentLocation = GetNextLocation(locationMode, currentLocation);
}
Assert.IsNull(retryPolicy.Evaluate(new RetryContext(expectedRetryInfoList.Length, requestResult, currentLocation, locationMode), operationContext));
}
