private static void RetryRequest <T, INTERMEDIATE_TYPE>(object state) { ExecutionState <T> executionState = (ExecutionState <T>)state; Logger.LogInformational(executionState.OperationContext, SR.TraceRetry); TableExecutor.InitRequest <T, INTERMEDIATE_TYPE>(executionState); }
private static void Context_SendingSignedRequest <T>(ExecutionState <T> executionState, HttpWebRequest req) { // Handle multiple astoria transactions per execute if (executionState.Req != null) { executionState.Resp = GetResponseForRequest(executionState.Req); TableExecutor.FireResponseReceived(executionState); executionState.Req = null; // Finish previous attempt TableExecutor.FinishRequestAttempt(executionState); // on successes for multiple Astoria operations start new request if ((int)executionState.Resp.StatusCode < 300) { // 0. Setup Next RequestResult TableExecutor.StartRequestAttempt(executionState); } TableExecutor.CheckTimeout <T>(executionState, true); } executionState.Req = req; TableExecutor.ApplyUserHeaders(executionState); TableExecutor.FireSendingRequest(executionState); }
// Cancellation is handeld by TableServiceContext public static ICancellableAsyncResult BeginExecuteAsync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext, AsyncCallback callback, object asyncState) { // Note all code below will reference state, not params directly, this will allow common code with async executor ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext, callback, asyncState); TableExecutor.AcquireContext(cmd.Context, executionState); InitRequest <T, INTERMEDIATE_TYPE>(executionState); return(executionState); }
private static void Context_SendingSignedRequest <T>(ExecutionState <T> executionState, HttpWebRequest req) { // Finish previous attempt. if (executionState.Req != null) { TableExecutor.FinishRequestAttempt(executionState); TableExecutor.StartRequestAttempt(executionState); TableExecutor.CheckTimeout <T>(executionState, true); } // Start new operation. executionState.Req = req; TableExecutor.ApplyUserHeaders(executionState); TableExecutor.FireSendingRequest(executionState); }
public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState) { try { executionState.Init(); // 0. Begin Request TableExecutor.StartRequestAttempt(executionState); if (TableExecutor.CheckTimeout <T>(executionState, false)) { TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState); return; } lock (executionState.CancellationLockerObject) { if (TableExecutor.CheckCancellation(executionState)) { TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState); return; } TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>; // Execute Call tableCommandRef.Begin( (res) => { executionState.CompletedSynchronously = executionState.CompletedSynchronously && res.CompletedSynchronously; INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE); try { tResult = tableCommandRef.End(res); executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef); // Attempt to populate response headers if (executionState.Req != null) { executionState.Resp = GetResponseForRequest(executionState.Req); FireResponseReceived(executionState); } } catch (Exception ex) { lock (executionState.CancellationLockerObject) { if (executionState.CancelRequested) { // Ignore DSC exception if request was canceled. return; } } // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult); } try { // Attempt to populate response headers if (executionState.Req != null) { executionState.Resp = GetResponseForRequest(executionState.Req); FireResponseReceived(executionState); } executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef); // clear exception executionState.ExceptionRef = null; } catch (Exception parseEx) { executionState.ExceptionRef = parseEx; } } finally { EndOperation <T, INTERMEDIATE_TYPE>(executionState); } }, null); if (tableCommandRef.Context != null) { executionState.CancelDelegate = tableCommandRef.Context.InternalCancel; } } } catch (Exception ex) { // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult); } executionState.OnComplete(); } }
public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext) { // Note all code below will reference state, not params directly, this will allow common code with async executor ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext); TableExecutor.AcquireContext(cmd.Context, executionState); bool shouldRetry = false; try { // Enter Retryable Section of execution do { executionState.Init(); // 0. Begin Request TableExecutor.StartRequestAttempt(executionState); TableExecutor.CheckTimeout <T>(executionState, true); try { INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE); try { tempResult = cmd.ExecuteFunc(); executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd); // Attempt to populate response headers if (executionState.Req != null) { executionState.Resp = GetResponseForRequest(executionState.Req); TableExecutor.FireResponseReceived(executionState); } } catch (Exception ex) { // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult); } // Attempt to populate response headers if (executionState.Req != null) { executionState.Resp = GetResponseForRequest(executionState.Req); TableExecutor.FireResponseReceived(executionState); } executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd); // clear exception executionState.ExceptionRef = null; } return(executionState.Result); } catch (Exception e) { TableExecutor.FinishRequestAttempt(executionState); StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult); executionState.ExceptionRef = translatedException; TimeSpan delay = TimeSpan.FromMilliseconds(0); shouldRetry = translatedException.IsRetryable && executionState.RetryPolicy != null? executionState.RetryPolicy.ShouldRetry( executionState.RetryCount++, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext) : false; delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay; if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0)) { throw executionState.ExceptionRef; } else { if (executionState.Cmd.RecoveryAction != null) { // I.E. Rewind stream etc. executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext); } if (delay > TimeSpan.Zero) { Thread.Sleep(delay); } } } }while (shouldRetry); // should never get here, either return, or throw; throw new NotImplementedException(SR.InternalStorageError); } finally { ReleaseContext(cmd.Context); } }
private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState) { TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>; TableExecutor.FinishRequestAttempt(executionState); lock (executionState.CancellationLockerObject) { executionState.CancelDelegate = null; TableExecutor.CheckCancellation(executionState); // Handle Success if (executionState.ExceptionRef == null) { // Signal event handled and mark response executionState.OnComplete(); return; } } // Handle Retry try { StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult); executionState.ExceptionRef = translatedException; TimeSpan delay = TimeSpan.FromMilliseconds(0); bool shouldRetry = translatedException.IsRetryable && executionState.RetryPolicy != null? executionState.RetryPolicy.ShouldRetry( executionState.RetryCount++, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext) : false; delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay; if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0)) { // No Retry executionState.OnComplete(); } else { if (executionState.Cmd.RecoveryAction != null) { // I.E. Rewind stream etc. executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext); } if (delay > TimeSpan.Zero) { Timer backoffTimer = null; backoffTimer = new Timer( (obj) => { backoffTimer.Change(Timeout.Infinite, Timeout.Infinite); backoffTimer.Dispose(); InitRequest <T, INTERMEDIATE_TYPE>(executionState); }, null /* state */, (int)delay.TotalMilliseconds, Timeout.Infinite); } else { // Start Next Request Immediately InitRequest <T, INTERMEDIATE_TYPE>(executionState); } } } catch (Exception ex) { // Catch all ( i.e. users retry policy throws etc.) executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult); executionState.OnComplete(); } }
public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState) { try { executionState.Init(); // 0. Begin Request TableExecutor.StartRequestAttempt(executionState); if (TableExecutor.CheckTimeout <T>(executionState, false)) { TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState); return; } lock (executionState.CancellationLockerObject) { if (TableExecutor.CheckCancellation(executionState)) { TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState); return; } TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>; // Execute Call Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, tableCommandRef.Context.BaseUri); tableCommandRef.Begin( (res) => { executionState.UpdateCompletedSynchronously(res.CompletedSynchronously); INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE); try { tResult = tableCommandRef.End(res); executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef); if (executionState.Req != null) { DataServiceResponse dataServiceResponse = tResult as DataServiceResponse; QueryOperationResponse queryResponse = tResult as QueryOperationResponse; if (dataServiceResponse != null) { if (dataServiceResponse.IsBatchResponse) { // Attempt to populate response headers if (executionState.Req != null) { SetExecutionStateCommandResult(executionState, dataServiceResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag); } } else { int index = 0; foreach (OperationResponse operationResponse in dataServiceResponse) { // Attempt to populate response headers if (executionState.Req != null) { SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag); index++; } } } } else if (queryResponse != null) { // Attempt to populate response headers if (executionState.Req != null) { SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag); } } } } catch (Exception ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message); lock (executionState.CancellationLockerObject) { if (executionState.CancelRequested) { // Ignore DSC exception if request was canceled. return; } } // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError); } try { executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef); // clear exception executionState.ExceptionRef = null; } catch (Exception parseEx) { Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message); executionState.ExceptionRef = parseEx; } } finally { EndOperation <T, INTERMEDIATE_TYPE>(executionState); } }, null); if (tableCommandRef.Context != null) { executionState.CancelDelegate = tableCommandRef.Context.InternalCancel; } } } catch (Exception ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message); // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError); } executionState.OnComplete(); } }
public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext) { // Note all code below will reference state, not params directly, this will allow common code with async executor ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext); TableExecutor.AcquireContext(cmd.Context, executionState); bool shouldRetry = false; TimeSpan delay = TimeSpan.Zero; try { // Enter Retryable Section of execution do { executionState.Init(); // 0. Begin Request TableExecutor.StartRequestAttempt(executionState); TableExecutor.CheckTimeout <T>(executionState, true); try { INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE); try { Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestSync, cmd.Context.BaseUri); tempResult = cmd.ExecuteFunc(); executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd); DataServiceResponse dataServiceResponse = tempResult as DataServiceResponse; QueryOperationResponse queryResponse = tempResult as QueryOperationResponse; if (dataServiceResponse != null) { if (dataServiceResponse.IsBatchResponse) { // Attempt to populate response headers if (executionState.Req != null) { SetExecutionStateCommandResult(executionState, dataServiceResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag); } } else { int index = 0; foreach (OperationResponse operationResponse in dataServiceResponse) { // Attempt to populate response headers if (executionState.Req != null) { SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag); index++; } } } } else if (queryResponse != null) { // Attempt to populate response headers if (executionState.Req != null) { SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse); Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag); } } } catch (Exception ex) { Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message); // Store exception and invoke callback here. All operations in this try would be non-retryable by default if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException)) { executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError); } executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd); // clear exception executionState.ExceptionRef = null; } TableExecutor.FinishRequestAttempt(executionState); Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess); return(executionState.Result); } catch (Exception e) { TableExecutor.FinishRequestAttempt(executionState); StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError); executionState.ExceptionRef = translatedException; Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message); shouldRetry = false; if (translatedException.IsRetryable && (executionState.RetryPolicy != null)) { shouldRetry = executionState.RetryPolicy.ShouldRetry( executionState.RetryCount++, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext); if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff)) { delay = Constants.MaximumRetryBackoff; } } } if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0)) { Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message); throw executionState.ExceptionRef; } else { if (executionState.Cmd.RecoveryAction != null) { // I.E. Rewind stream etc. executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext); } Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds); if (delay > TimeSpan.Zero) { Thread.Sleep(delay); } Logger.LogInformational(executionState.OperationContext, SR.TraceRetry); } }while (shouldRetry); // should never get here, either return, or throw; throw new NotImplementedException(SR.InternalStorageError); } finally { ReleaseContext(cmd.Context); } }
private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState) { TableExecutor.FinishRequestAttempt(executionState); lock (executionState.CancellationLockerObject) { executionState.CancelDelegate = null; TableExecutor.CheckCancellation(executionState); // Handle Success if (executionState.ExceptionRef == null) { Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess); executionState.OnComplete(); return; } } // Handle Retry try { StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError); executionState.ExceptionRef = translatedException; Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message); bool shouldRetry = false; TimeSpan delay = TimeSpan.Zero; if (translatedException.IsRetryable && (executionState.RetryPolicy != null)) { shouldRetry = executionState.RetryPolicy.ShouldRetry( executionState.RetryCount++, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.ExceptionRef, out delay, executionState.OperationContext); if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff)) { delay = Constants.MaximumRetryBackoff; } } if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0)) { Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message); // No Retry executionState.OnComplete(); } else { if (executionState.Cmd.RecoveryAction != null) { // I.E. Rewind stream etc. executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext); } if (delay > TimeSpan.Zero) { Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds); executionState.UpdateCompletedSynchronously(false); if (executionState.BackoffTimer == null) { executionState.BackoffTimer = new Timer( TableExecutor.RetryRequest <T, INTERMEDIATE_TYPE>, executionState, (int)delay.TotalMilliseconds, Timeout.Infinite); } else { executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite); } executionState.CancelDelegate = () => { // Disabling the timer here, but there is still a scenario where the user calls cancel after // the timer starts the next retry but before it sets the CancelDelegate back to null. However, even // if that happens, next retry will start and then stop immediately because of the cancelled flag. Timer backoffTimer = executionState.BackoffTimer; if (backoffTimer != null) { executionState.BackoffTimer = null; backoffTimer.Dispose(); } Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry); TableExecutor.CheckCancellation(executionState); executionState.OnComplete(); }; } else { // Start Next Request Immediately Logger.LogInformational(executionState.OperationContext, SR.TraceRetry); InitRequest <T, INTERMEDIATE_TYPE>(executionState); } } } catch (Exception ex) { // Catch all ( i.e. users retry policy throws etc.) Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message); executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError); executionState.OnComplete(); } }