private static void RetryRequest <T, INTERMEDIATE_TYPE>(object state)
{
ExecutionState <T> executionState = (ExecutionState <T>)state;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
TableExecutor.InitRequest <T, INTERMEDIATE_TYPE>(executionState);
}
// Cancellation is handeld by TableServiceContext
public static ICancellableAsyncResult BeginExecuteAsync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext, AsyncCallback callback, object asyncState)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext, callback, asyncState);
TableExecutor.AcquireContext(cmd.Context, executionState);
InitRequest <T, INTERMEDIATE_TYPE>(executionState);
return(executionState);
}
private static void Context_SendingSignedRequest <T>(ExecutionState <T> executionState, HttpWebRequest req)
{
// Finish previous attempt.
if (executionState.Req != null)
{
TableExecutor.FinishRequestAttempt(executionState);
TableExecutor.StartRequestAttempt(executionState);
TableExecutor.CheckTimeout <T>(executionState, true);
}
// Start new operation.
executionState.Req = req;
TableExecutor.ApplyUserHeaders(executionState);
TableExecutor.FireSendingRequest(executionState);
}
public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
{
try
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
if (TableExecutor.CheckTimeout <T>(executionState, false))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
lock (executionState.CancellationLockerObject)
{
if (TableExecutor.CheckCancellation(executionState))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;
// Execute Call
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, tableCommandRef.Context.BaseUri);
tableCommandRef.Begin(
(res) =>
{
executionState.UpdateCompletedSynchronously(res.CompletedSynchronously);
INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE);
try
{
tResult = tableCommandRef.End(res);
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
if (executionState.Req != null)
{
DataServiceResponse dataServiceResponse = tResult as DataServiceResponse;
QueryOperationResponse queryResponse = tResult as QueryOperationResponse;
if (dataServiceResponse != null)
{
if (dataServiceResponse.IsBatchResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetExecutionStateCommandResult(executionState, dataServiceResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
else
{
int index = 0;
foreach (OperationResponse operationResponse in dataServiceResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
index++;
}
}
}
}
else if (queryResponse != null)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
lock (executionState.CancellationLockerObject)
{
if (executionState.CancelRequested)
{
// Ignore DSC exception if request was canceled.
return;
}
}
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
try
{
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception parseEx)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
executionState.ExceptionRef = parseEx;
}
}
finally
{
EndOperation <T, INTERMEDIATE_TYPE>(executionState);
}
},
null);
if (tableCommandRef.Context != null)
{
executionState.CancelDelegate = tableCommandRef.Context.InternalCancel;
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
executionState.OnComplete();
}
}
public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);
TableExecutor.AcquireContext(cmd.Context, executionState);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
try
{
// Enter Retryable Section of execution
do
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
TableExecutor.CheckTimeout <T>(executionState, true);
try
{
INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE);
try
{
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestSync, cmd.Context.BaseUri);
tempResult = cmd.ExecuteFunc();
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
DataServiceResponse dataServiceResponse = tempResult as DataServiceResponse;
QueryOperationResponse queryResponse = tempResult as QueryOperationResponse;
if (dataServiceResponse != null)
{
if (dataServiceResponse.IsBatchResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetExecutionStateCommandResult(executionState, dataServiceResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
else
{
int index = 0;
foreach (OperationResponse operationResponse in dataServiceResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
index++;
}
}
}
}
else if (queryResponse != null)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
// clear exception
executionState.ExceptionRef = null;
}
TableExecutor.FinishRequestAttempt(executionState);
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
return(executionState.Result);
}
catch (Exception e)
{
TableExecutor.FinishRequestAttempt(executionState);
StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
if (delay > TimeSpan.Zero)
{
Thread.Sleep(delay);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
}while (shouldRetry);
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
finally
{
ReleaseContext(cmd.Context);
}
}
private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
{
TableExecutor.FinishRequestAttempt(executionState);
lock (executionState.CancellationLockerObject)
{
executionState.CancelDelegate = null;
TableExecutor.CheckCancellation(executionState);
// Handle Success
if (executionState.ExceptionRef == null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
executionState.OnComplete();
return;
}
}
// Handle Retry
try
{
StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
executionState.UpdateCompletedSynchronously(false);
if (executionState.BackoffTimer == null)
{
executionState.BackoffTimer = new Timer(
TableExecutor.RetryRequest <T, INTERMEDIATE_TYPE>,
executionState,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
}
executionState.CancelDelegate = () =>
{
// Disabling the timer here, but there is still a scenario where the user calls cancel after
// the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
// if that happens, next retry will start and then stop immediately because of the cancelled flag.
Timer backoffTimer = executionState.BackoffTimer;
if (backoffTimer != null)
{
executionState.BackoffTimer = null;
backoffTimer.Dispose();
}
Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
TableExecutor.CheckCancellation(executionState);
executionState.OnComplete();
};
}
else
{
// Start Next Request Immediately
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
InitRequest <T, INTERMEDIATE_TYPE>(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.OnComplete();
}
}
