コード例 #1
0
        private static void RetryRequest <T, INTERMEDIATE_TYPE>(object state)
        {
            ExecutionState <T> executionState = (ExecutionState <T>)state;

            Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
            TableExecutor.InitRequest <T, INTERMEDIATE_TYPE>(executionState);
        }
コード例 #2
0
        private static void Context_SendingSignedRequest <T>(ExecutionState <T> executionState, HttpWebRequest req)
        {
            // Handle multiple astoria transactions per execute
            if (executionState.Req != null)
            {
                executionState.Resp = GetResponseForRequest(executionState.Req);

                TableExecutor.FireResponseReceived(executionState);

                executionState.Req = null;

                // Finish previous attempt
                TableExecutor.FinishRequestAttempt(executionState);

                // on successes for multiple Astoria operations start new request
                if ((int)executionState.Resp.StatusCode < 300)
                {
                    // 0. Setup Next RequestResult
                    TableExecutor.StartRequestAttempt(executionState);
                }

                TableExecutor.CheckTimeout <T>(executionState, true);
            }

            executionState.Req = req;
            TableExecutor.ApplyUserHeaders(executionState);
            TableExecutor.FireSendingRequest(executionState);
        }
コード例 #3
0
        // Cancellation is handeld by TableServiceContext
        public static ICancellableAsyncResult BeginExecuteAsync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext, AsyncCallback callback, object asyncState)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext, callback, asyncState);

            TableExecutor.AcquireContext(cmd.Context, executionState);
            InitRequest <T, INTERMEDIATE_TYPE>(executionState);
            return(executionState);
        }
コード例 #4
0
        private static void Context_SendingSignedRequest <T>(ExecutionState <T> executionState, HttpWebRequest req)
        {
            // Finish previous attempt.
            if (executionState.Req != null)
            {
                TableExecutor.FinishRequestAttempt(executionState);
                TableExecutor.StartRequestAttempt(executionState);
                TableExecutor.CheckTimeout <T>(executionState, true);
            }

            // Start new operation.
            executionState.Req = req;

            TableExecutor.ApplyUserHeaders(executionState);
            TableExecutor.FireSendingRequest(executionState);
        }
コード例 #5
0
        public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
        {
            try
            {
                executionState.Init();

                // 0. Begin Request
                TableExecutor.StartRequestAttempt(executionState);

                if (TableExecutor.CheckTimeout <T>(executionState, false))
                {
                    TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                    return;
                }

                lock (executionState.CancellationLockerObject)
                {
                    if (TableExecutor.CheckCancellation(executionState))
                    {
                        TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                        return;
                    }

                    TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;

                    // Execute Call
                    tableCommandRef.Begin(
                        (res) =>
                    {
                        executionState.CompletedSynchronously = executionState.CompletedSynchronously && res.CompletedSynchronously;
                        INTERMEDIATE_TYPE tResult             = default(INTERMEDIATE_TYPE);

                        try
                        {
                            tResult = tableCommandRef.End(res);

                            executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);

                            // Attempt to populate response headers
                            if (executionState.Req != null)
                            {
                                executionState.Resp = GetResponseForRequest(executionState.Req);
                                FireResponseReceived(executionState);
                            }
                        }
                        catch (Exception ex)
                        {
                            lock (executionState.CancellationLockerObject)
                            {
                                if (executionState.CancelRequested)
                                {
                                    // Ignore DSC exception if request was canceled.
                                    return;
                                }
                            }

                            // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                            if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                            {
                                executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                            }

                            try
                            {
                                // Attempt to populate response headers
                                if (executionState.Req != null)
                                {
                                    executionState.Resp = GetResponseForRequest(executionState.Req);
                                    FireResponseReceived(executionState);
                                }

                                executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);

                                // clear exception
                                executionState.ExceptionRef = null;
                            }
                            catch (Exception parseEx)
                            {
                                executionState.ExceptionRef = parseEx;
                            }
                        }
                        finally
                        {
                            EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                        }
                    },
                        null);

                    if (tableCommandRef.Context != null)
                    {
                        executionState.CancelDelegate = tableCommandRef.Context.InternalCancel;
                    }
                }
            }
            catch (Exception ex)
            {
                // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                {
                    executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                }

                executionState.OnComplete();
            }
        }
コード例 #6
0
        public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);

            TableExecutor.AcquireContext(cmd.Context, executionState);
            bool shouldRetry = false;

            try
            {
                // Enter Retryable Section of execution
                do
                {
                    executionState.Init();

                    // 0. Begin Request
                    TableExecutor.StartRequestAttempt(executionState);

                    TableExecutor.CheckTimeout <T>(executionState, true);

                    try
                    {
                        INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE);

                        try
                        {
                            tempResult = cmd.ExecuteFunc();

                            executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);

                            // Attempt to populate response headers
                            if (executionState.Req != null)
                            {
                                executionState.Resp = GetResponseForRequest(executionState.Req);
                                TableExecutor.FireResponseReceived(executionState);
                            }
                        }
                        catch (Exception ex)
                        {
                            // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                            if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                            {
                                executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                            }

                            // Attempt to populate response headers
                            if (executionState.Req != null)
                            {
                                executionState.Resp = GetResponseForRequest(executionState.Req);
                                TableExecutor.FireResponseReceived(executionState);
                            }

                            executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);

                            // clear exception
                            executionState.ExceptionRef = null;
                        }

                        return(executionState.Result);
                    }
                    catch (Exception e)
                    {
                        TableExecutor.FinishRequestAttempt(executionState);

                        StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
                        executionState.ExceptionRef = translatedException;

                        TimeSpan delay = TimeSpan.FromMilliseconds(0);
                        shouldRetry = translatedException.IsRetryable &&
                                      executionState.RetryPolicy != null?
                                      executionState.RetryPolicy.ShouldRetry(
                            executionState.RetryCount++,
                            executionState.Cmd.CurrentResult.HttpStatusCode,
                            executionState.ExceptionRef,
                            out delay,
                            executionState.OperationContext)
                                          : false;

                        delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;

                        if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                        {
                            throw executionState.ExceptionRef;
                        }
                        else
                        {
                            if (executionState.Cmd.RecoveryAction != null)
                            {
                                // I.E. Rewind stream etc.
                                executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                            }

                            if (delay > TimeSpan.Zero)
                            {
                                Thread.Sleep(delay);
                            }
                        }
                    }
                }while (shouldRetry);

                // should never get here, either return, or throw;
                throw new NotImplementedException(SR.InternalStorageError);
            }
            finally
            {
                ReleaseContext(cmd.Context);
            }
        }
コード例 #7
0
        private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
        {
            TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;

            TableExecutor.FinishRequestAttempt(executionState);

            lock (executionState.CancellationLockerObject)
            {
                executionState.CancelDelegate = null;

                TableExecutor.CheckCancellation(executionState);

                // Handle Success
                if (executionState.ExceptionRef == null)
                {
                    // Signal event handled and mark response
                    executionState.OnComplete();
                    return;
                }
            }

            // Handle Retry
            try
            {
                StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
                executionState.ExceptionRef = translatedException;

                TimeSpan delay       = TimeSpan.FromMilliseconds(0);
                bool     shouldRetry = translatedException.IsRetryable &&
                                       executionState.RetryPolicy != null?
                                       executionState.RetryPolicy.ShouldRetry(
                    executionState.RetryCount++,
                    executionState.Cmd.CurrentResult.HttpStatusCode,
                    executionState.ExceptionRef,
                    out delay,
                    executionState.OperationContext)
                                           : false;

                delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;

                if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                {
                    // No Retry
                    executionState.OnComplete();
                }
                else
                {
                    if (executionState.Cmd.RecoveryAction != null)
                    {
                        // I.E. Rewind stream etc.
                        executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                    }

                    if (delay > TimeSpan.Zero)
                    {
                        Timer backoffTimer = null;

                        backoffTimer = new Timer(
                            (obj) =>
                        {
                            backoffTimer.Change(Timeout.Infinite, Timeout.Infinite);
                            backoffTimer.Dispose();
                            InitRequest <T, INTERMEDIATE_TYPE>(executionState);
                        },
                            null /* state */,
                            (int)delay.TotalMilliseconds,
                            Timeout.Infinite);
                    }
                    else
                    {
                        // Start Next Request Immediately
                        InitRequest <T, INTERMEDIATE_TYPE>(executionState);
                    }
                }
            }
            catch (Exception ex)
            {
                // Catch all ( i.e. users retry policy throws etc.)
                executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                executionState.OnComplete();
            }
        }
コード例 #8
0
        public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
        {
            try
            {
                executionState.Init();

                // 0. Begin Request
                TableExecutor.StartRequestAttempt(executionState);

                if (TableExecutor.CheckTimeout <T>(executionState, false))
                {
                    TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                    return;
                }

                lock (executionState.CancellationLockerObject)
                {
                    if (TableExecutor.CheckCancellation(executionState))
                    {
                        TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                        return;
                    }

                    TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;

                    // Execute Call
                    Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, tableCommandRef.Context.BaseUri);
                    tableCommandRef.Begin(
                        (res) =>
                    {
                        executionState.UpdateCompletedSynchronously(res.CompletedSynchronously);
                        INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE);

                        try
                        {
                            tResult = tableCommandRef.End(res);

                            executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);

                            if (executionState.Req != null)
                            {
                                DataServiceResponse dataServiceResponse = tResult as DataServiceResponse;
                                QueryOperationResponse queryResponse    = tResult as QueryOperationResponse;

                                if (dataServiceResponse != null)
                                {
                                    if (dataServiceResponse.IsBatchResponse)
                                    {
                                        // Attempt to populate response headers
                                        if (executionState.Req != null)
                                        {
                                            SetExecutionStateCommandResult(executionState, dataServiceResponse);
                                            Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                                        }
                                    }
                                    else
                                    {
                                        int index = 0;
                                        foreach (OperationResponse operationResponse in dataServiceResponse)
                                        {
                                            // Attempt to populate response headers
                                            if (executionState.Req != null)
                                            {
                                                SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
                                                Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
                                                index++;
                                            }
                                        }
                                    }
                                }
                                else if (queryResponse != null)
                                {
                                    // Attempt to populate response headers
                                    if (executionState.Req != null)
                                    {
                                        SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);

                            lock (executionState.CancellationLockerObject)
                            {
                                if (executionState.CancelRequested)
                                {
                                    // Ignore DSC exception if request was canceled.
                                    return;
                                }
                            }

                            // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                            if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                            {
                                executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
                            }

                            try
                            {
                                executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);

                                // clear exception
                                executionState.ExceptionRef = null;
                            }
                            catch (Exception parseEx)
                            {
                                Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
                                executionState.ExceptionRef = parseEx;
                            }
                        }
                        finally
                        {
                            EndOperation <T, INTERMEDIATE_TYPE>(executionState);
                        }
                    },
                        null);

                    if (tableCommandRef.Context != null)
                    {
                        executionState.CancelDelegate = tableCommandRef.Context.InternalCancel;
                    }
                }
            }
            catch (Exception ex)
            {
                Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);

                // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                {
                    executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
                }

                executionState.OnComplete();
            }
        }
コード例 #9
0
        public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);

            TableExecutor.AcquireContext(cmd.Context, executionState);
            bool     shouldRetry = false;
            TimeSpan delay       = TimeSpan.Zero;

            try
            {
                // Enter Retryable Section of execution
                do
                {
                    executionState.Init();

                    // 0. Begin Request
                    TableExecutor.StartRequestAttempt(executionState);

                    TableExecutor.CheckTimeout <T>(executionState, true);

                    try
                    {
                        INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE);

                        try
                        {
                            Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestSync, cmd.Context.BaseUri);
                            tempResult = cmd.ExecuteFunc();

                            executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);

                            DataServiceResponse    dataServiceResponse = tempResult as DataServiceResponse;
                            QueryOperationResponse queryResponse       = tempResult as QueryOperationResponse;

                            if (dataServiceResponse != null)
                            {
                                if (dataServiceResponse.IsBatchResponse)
                                {
                                    // Attempt to populate response headers
                                    if (executionState.Req != null)
                                    {
                                        SetExecutionStateCommandResult(executionState, dataServiceResponse);
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                                    }
                                }
                                else
                                {
                                    int index = 0;
                                    foreach (OperationResponse operationResponse in dataServiceResponse)
                                    {
                                        // Attempt to populate response headers
                                        if (executionState.Req != null)
                                        {
                                            SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
                                            Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
                                            index++;
                                        }
                                    }
                                }
                            }
                            else if (queryResponse != null)
                            {
                                // Attempt to populate response headers
                                if (executionState.Req != null)
                                {
                                    SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
                                    Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);

                            // Store exception and invoke callback here. All operations in this try would be non-retryable by default
                            if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
                            {
                                executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
                            }

                            executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);

                            // clear exception
                            executionState.ExceptionRef = null;
                        }

                        TableExecutor.FinishRequestAttempt(executionState);

                        Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
                        return(executionState.Result);
                    }
                    catch (Exception e)
                    {
                        TableExecutor.FinishRequestAttempt(executionState);

                        StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
                        executionState.ExceptionRef = translatedException;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                        shouldRetry = false;
                        if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                        {
                            shouldRetry = executionState.RetryPolicy.ShouldRetry(
                                executionState.RetryCount++,
                                executionState.Cmd.CurrentResult.HttpStatusCode,
                                executionState.ExceptionRef,
                                out delay,
                                executionState.OperationContext);

                            if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                            {
                                delay = Constants.MaximumRetryBackoff;
                            }
                        }
                    }

                    if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                    {
                        Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
                        throw executionState.ExceptionRef;
                    }
                    else
                    {
                        if (executionState.Cmd.RecoveryAction != null)
                        {
                            // I.E. Rewind stream etc.
                            executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                        }

                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
                        if (delay > TimeSpan.Zero)
                        {
                            Thread.Sleep(delay);
                        }

                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                    }
                }while (shouldRetry);

                // should never get here, either return, or throw;
                throw new NotImplementedException(SR.InternalStorageError);
            }
            finally
            {
                ReleaseContext(cmd.Context);
            }
        }
コード例 #10
0
        private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
        {
            TableExecutor.FinishRequestAttempt(executionState);

            lock (executionState.CancellationLockerObject)
            {
                executionState.CancelDelegate = null;

                TableExecutor.CheckCancellation(executionState);

                // Handle Success
                if (executionState.ExceptionRef == null)
                {
                    Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
                    executionState.OnComplete();
                    return;
                }
            }

            // Handle Retry
            try
            {
                StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);

                executionState.ExceptionRef = translatedException;
                Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                bool     shouldRetry = false;
                TimeSpan delay       = TimeSpan.Zero;
                if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                {
                    shouldRetry = executionState.RetryPolicy.ShouldRetry(
                        executionState.RetryCount++,
                        executionState.Cmd.CurrentResult.HttpStatusCode,
                        executionState.ExceptionRef,
                        out delay,
                        executionState.OperationContext);

                    if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                    {
                        delay = Constants.MaximumRetryBackoff;
                    }
                }

                if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                {
                    Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);

                    // No Retry
                    executionState.OnComplete();
                }
                else
                {
                    if (executionState.Cmd.RecoveryAction != null)
                    {
                        // I.E. Rewind stream etc.
                        executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                    }

                    if (delay > TimeSpan.Zero)
                    {
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);

                        executionState.UpdateCompletedSynchronously(false);
                        if (executionState.BackoffTimer == null)
                        {
                            executionState.BackoffTimer = new Timer(
                                TableExecutor.RetryRequest <T, INTERMEDIATE_TYPE>,
                                executionState,
                                (int)delay.TotalMilliseconds,
                                Timeout.Infinite);
                        }
                        else
                        {
                            executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
                        }

                        executionState.CancelDelegate = () =>
                        {
                            // Disabling the timer here, but there is still a scenario where the user calls cancel after
                            // the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
                            // if that happens, next retry will start and then stop immediately because of the cancelled flag.
                            Timer backoffTimer = executionState.BackoffTimer;
                            if (backoffTimer != null)
                            {
                                executionState.BackoffTimer = null;
                                backoffTimer.Dispose();
                            }

                            Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
                            TableExecutor.CheckCancellation(executionState);
                            executionState.OnComplete();
                        };
                    }
                    else
                    {
                        // Start Next Request Immediately
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                        InitRequest <T, INTERMEDIATE_TYPE>(executionState);
                    }
                }
            }
            catch (Exception ex)
            {
                // Catch all ( i.e. users retry policy throws etc.)
                Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
                executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);

                executionState.OnComplete();
            }
        }