protected static void FireResponseReceived <T>(ExecutionState <T> executionState)
{
RequestEventArgs args = GenerateRequestEventArgs <T>(executionState);
executionState.OperationContext.FireResponseReceived(args);
}
protected static void FireRetrying <T>(ExecutionState <T> executionState)
{
RequestEventArgs args = GenerateRequestEventArgs <T>(executionState);
executionState.OperationContext.FireRetrying(args);
}
public static T ExecuteSync <T>(StorageCommandBase <T> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// Steps 1-4
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestSync, cmd.Uri);
Executor.ProcessStartOfRequest(executionState);
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default
StorageException storageEx = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// 5. potentially upload data
if (executionState.RestCMD.SendStream != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginGetRequestStream;
Logger.LogInformational(executionState.OperationContext, SR.TracePrepareUpload);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.ReqStream = executionState.Req.GetRequestStream();
executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest;
Logger.LogInformational(executionState.OperationContext, SR.TraceUpload);
MultiBufferMemoryStream multiBufferMemoryStream = executionState.RestCMD.SendStream as MultiBufferMemoryStream;
try
{
if (multiBufferMemoryStream != null && !executionState.RestCMD.SendStreamLength.HasValue)
{
multiBufferMemoryStream.FastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime);
}
else
{
// don't calculate md5 here as we should have already set this for auth purposes
executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, executionState.RestCMD.SendStreamLength, null /* maxLength */, false, true, executionState, null /* streamCopyState */);
}
executionState.ReqStream.Flush();
executionState.ReqStream.Dispose();
executionState.ReqStream = null;
}
catch (Exception)
{
executionState.Req.Abort();
throw;
}
}
// 6. Get response
try
{
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
executionState.Resp = (HttpWebResponse)executionState.Req.GetResponse();
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
}
catch (WebException ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
executionState.Resp = (HttpWebResponse)ex.Response;
if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
{
throw new TimeoutException();
}
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
}
// Response
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (executionState.RestCMD.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();
if (!executionState.RestCMD.RetrieveResponseStream)
{
executionState.RestCMD.DestinationStream = Stream.Null;
}
if (executionState.RestCMD.DestinationStream != null)
{
if (executionState.RestCMD.StreamCopyState == null)
{
executionState.RestCMD.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
executionState.RestCMD.ResponseStream.WriteToSync(executionState.RestCMD.DestinationStream, null /* copyLength */, null /* maxLength */, executionState.RestCMD.CalculateMd5ForResponseStream, false, executionState, executionState.RestCMD.StreamCopyState);
}
finally
{
executionState.RestCMD.ResponseStream.Dispose();
executionState.RestCMD.ResponseStream = null;
}
}
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
Thread.Sleep(delay);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
}while (shouldRetry);
}
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
private async static Task <T> ExecuteAsyncInternal <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
{
// Note all code below will reference state, not params directly, this will allow common code with multiple executors (APM, Sync, Async)
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
// Note - The service accepts both api-version and x-ms-version and therefore it is ok to add x-ms-version to all requests.
// Create a new client
HttpClient client = HttpClientFactory.Instance;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// 1. Build request and content
executionState.CurrentOperation = ExecutorOperation.BeginOperation;
// Content is re-created every retry, as HttpClient disposes it after a successful request
HttpContent content = cmd.BuildContent != null?cmd.BuildContent(cmd, executionState.OperationContext) : null;
// This is so the old auth header etc is cleared out, the content is where serialization occurs which is the major perf hit
Uri uri = cmd.StorageUri.GetUri(executionState.CurrentLocation);
Uri transformedUri = cmd.Credentials.TransformUri(uri);
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, transformedUri);
UriQueryBuilder builder = new UriQueryBuilder(executionState.RestCMD.Builder);
executionState.Req = cmd.BuildRequest(cmd, transformedUri, builder, content, cmd.ServerTimeoutInSeconds, executionState.OperationContext);
// 2. Set Headers
Executor.ApplyUserHeaders(executionState);
// Let the user know we are ready to send
Executor.FireSendingRequest(executionState);
// 3. Sign Request is not needed, as HttpClient will call us
// 4. Set timeout
if (executionState.OperationExpiryTime.HasValue)
{
// set the token to cancel after timing out, if the higher token hasn't already been cancelled
timeoutTokenSource.CancelAfter(executionState.RemainingTimeout);
}
else
{
// effectively prevent timeout
timeoutTokenSource.CancelAfter(int.MaxValue);
}
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default
StorageException storageEx = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult, null);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
throw executionState.ExceptionRef;
}
// Enter Retryable Section of execution
try
{
// Send Request
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutTokenSource.Token).ConfigureAwait(false);
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
// Since HttpClient wont throw for non success, manually check and populate an exception
if (!executionState.Resp.IsSuccessStatusCode)
{
// At this point, don't try to read the stream to parse the error
executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, executionState.Cmd.ParseError).ConfigureAwait(false);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
try
{
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception ex)
{
executionState.ExceptionRef = ex;
}
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = await executionState.Resp.Content.ReadAsStreamAsync().ConfigureAwait(false);
// The stream is now available in ResponseStream. Use the stream to parse out the response or error
if (executionState.ExceptionRef != null)
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);
try
{
cmd.ErrorStream = new MemoryStream();
await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, null /* copyLength */, null /* maxLength */, false, executionState, new StreamDescriptor(), timeoutTokenSource.Token).ConfigureAwait(false);
cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
#if NETCORE
executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream, executionState.Resp);
#else
executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream);
#endif
throw executionState.ExceptionRef;
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
cmd.ErrorStream.Dispose();
cmd.ErrorStream = null;
}
}
else
{
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, executionState, cmd.StreamCopyState, timeoutTokenSource.Token).ConfigureAwait(false);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
}
// 9. Evaluate Response & Parse Results, (Stream potentially available here)
if (cmd.PostProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PostProcess;
Logger.LogInformational(executionState.OperationContext, SR.TracePostProcess);
executionState.Result = await cmd.PostProcessResponse(cmd, executionState.Resp, executionState.OperationContext).ConfigureAwait(false);
}
executionState.CurrentOperation = ExecutorOperation.EndOperation;
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
if (e is TaskCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
e = new TimeoutException(SR.TimeoutExceptionMessage, e);
}
#if NETCORE
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), executionState.Resp);
#else
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null));
#endif
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
cmd.CurrentResult,
executionState.CurrentLocation,
cmd.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
cmd.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Dispose();
executionState.Resp = null;
}
}
// potentially backoff
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
cmd.RecoveryAction(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay, token).ConfigureAwait(false);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
Executor.FireRetrying(executionState);
}while (shouldRetry);
// should never get here
throw new NotImplementedException(SR.InternalStorageError);
}
}
public static T ExecuteSync <T>(StorageCommandBase <T> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);
bool shouldRetry = false;
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// Steps 1-4
Executor.ProcessStartOfRequest(executionState);
Executor.CheckTimeout <T>(executionState, true);
// Enter Retryable Section of execution
// 5. potentially upload data
if (executionState.RestCMD.SendStream != null)
{
// Reset timeout
executionState.Req.Timeout = executionState.RemainingTimeout;
executionState.ReqStream = executionState.Req.GetRequestStream();
executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, null /* maxLength */, executionState.OperationExpiryTime, false, true, executionState.OperationContext, null /* streamCopyState */); // don't calculate md5 here as we should have already set this for auth purposes
}
// 6. Get response
try
{
// Reset timeout
executionState.Req.Timeout = executionState.RemainingTimeout;
executionState.Resp = (HttpWebResponse)executionState.Req.GetResponse();
}
catch (WebException ex)
{
executionState.Resp = (HttpWebResponse)ex.Response;
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
}
// Response
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (executionState.RestCMD.PreProcessResponse != null)
{
executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
}
// 8. (Potentially reads stream from server)
if (executionState.RestCMD.RetrieveResponseStream)
{
executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();
}
if (executionState.RestCMD.DestinationStream != null)
{
try
{
if (executionState.RestCMD.StreamCopyState == null)
{
executionState.RestCMD.StreamCopyState = new StreamDescriptor();
}
executionState.RestCMD.ResponseStream.WriteToSync(executionState.RestCMD.DestinationStream, null /* maxLength */, executionState.OperationExpiryTime, executionState.RestCMD.CalculateMd5ForResponseStream, false, executionState.OperationContext, executionState.RestCMD.StreamCopyState);
}
finally
{
executionState.RestCMD.ResponseStream.Dispose();
}
}
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState, executionState.ExceptionRef);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Executor.FinishRequestAttempt(executionState);
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
TimeSpan delay = TimeSpan.FromMilliseconds(0);
shouldRetry = translatedException.IsRetryable &&
executionState.RetryPolicy != null?
executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext)
: false;
delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Thread.Sleep(delay);
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
}while (shouldRetry);
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
private static void EndOperationWithPostProcess <T>(ExecutionState <T> executionState)
{
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
EndOperationAsync <T>(executionState, true); // Losing the task (fire-and-forget). Same concept as the 'Begin' methods in this class which return void.
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);
TableExecutor.AcquireContext(cmd.Context, executionState);
bool shouldRetry = false;
try
{
// Enter Retryable Section of execution
do
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
TableExecutor.CheckTimeout <T>(executionState, true);
try
{
INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE);
try
{
tempResult = cmd.ExecuteFunc();
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
// Attempt to populate response headers
if (executionState.Req != null)
{
executionState.Resp = GetResponseForRequest(executionState.Req);
TableExecutor.FireResponseReceived(executionState);
}
}
catch (Exception ex)
{
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
// Attempt to populate response headers
if (executionState.Req != null)
{
executionState.Resp = GetResponseForRequest(executionState.Req);
TableExecutor.FireResponseReceived(executionState);
}
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
// clear exception
executionState.ExceptionRef = null;
}
TableExecutor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
TableExecutor.FinishRequestAttempt(executionState);
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
TimeSpan delay = TimeSpan.FromMilliseconds(0);
shouldRetry = translatedException.IsRetryable &&
executionState.RetryPolicy != null?
executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext)
: false;
delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Thread.Sleep(delay);
}
}
}
}while (shouldRetry);
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
finally
{
ReleaseContext(cmd.Context);
}
}
private static void EndResponseStreamCopy <T>(ExecutionState <T> executionState)
{
EndOperation(executionState);
}
public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
{
try
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
if (TableExecutor.CheckTimeout <T>(executionState, false))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
lock (executionState.CancellationLockerObject)
{
if (TableExecutor.CheckCancellation(executionState))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;
// Execute Call
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, tableCommandRef.Context.BaseUri);
tableCommandRef.Begin(
(res) =>
{
executionState.UpdateCompletedSynchronously(res.CompletedSynchronously);
INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE);
try
{
tResult = tableCommandRef.End(res);
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
if (executionState.Req != null)
{
DataServiceResponse dataServiceResponse = tResult as DataServiceResponse;
QueryOperationResponse queryResponse = tResult as QueryOperationResponse;
if (dataServiceResponse != null)
{
if (dataServiceResponse.IsBatchResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetExecutionStateCommandResult(executionState, dataServiceResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
else
{
int index = 0;
foreach (OperationResponse operationResponse in dataServiceResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
index++;
}
}
}
}
else if (queryResponse != null)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
lock (executionState.CancellationLockerObject)
{
if (executionState.CancelRequested)
{
// Ignore DSC exception if request was canceled.
return;
}
}
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
try
{
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception parseEx)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
executionState.ExceptionRef = parseEx;
}
}
finally
{
EndOperation <T, INTERMEDIATE_TYPE>(executionState);
}
},
null);
if (tableCommandRef.Context != null)
{
executionState.CancelDelegate = tableCommandRef.Context.InternalCancel;
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
executionState.OnComplete();
}
}
private static void EndOperation <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
{
TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;
TableExecutor.FinishRequestAttempt(executionState);
lock (executionState.CancellationLockerObject)
{
executionState.CancelDelegate = null;
TableExecutor.CheckCancellation(executionState);
// Handle Success
if (executionState.ExceptionRef == null)
{
// Signal event handled and mark response
executionState.OnComplete();
return;
}
}
// Handle Retry
try
{
StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
TimeSpan delay = TimeSpan.FromMilliseconds(0);
bool shouldRetry = translatedException.IsRetryable &&
executionState.RetryPolicy != null?
executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext)
: false;
delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Timer backoffTimer = null;
backoffTimer = new Timer(
(obj) =>
{
backoffTimer.Change(Timeout.Infinite, Timeout.Infinite);
backoffTimer.Dispose();
InitRequest <T, INTERMEDIATE_TYPE>(executionState);
},
null /* state */,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
// Start Next Request Immediately
InitRequest <T, INTERMEDIATE_TYPE>(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
executionState.OnComplete();
}
}
public static T ExecuteSync <T, INTERMEDIATE_TYPE>(TableCommand <T, INTERMEDIATE_TYPE> cmd, IRetryPolicy policy, OperationContext operationContext)
{
// Note all code below will reference state, not params directly, this will allow common code with async executor
ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);
TableExecutor.AcquireContext(cmd.Context, executionState);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
try
{
// Enter Retryable Section of execution
do
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
TableExecutor.CheckTimeout <T>(executionState, true);
try
{
INTERMEDIATE_TYPE tempResult = default(INTERMEDIATE_TYPE);
try
{
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestSync, cmd.Context.BaseUri);
tempResult = cmd.ExecuteFunc();
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
DataServiceResponse dataServiceResponse = tempResult as DataServiceResponse;
QueryOperationResponse queryResponse = tempResult as QueryOperationResponse;
if (dataServiceResponse != null)
{
if (dataServiceResponse.IsBatchResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetExecutionStateCommandResult(executionState, dataServiceResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
else
{
int index = 0;
foreach (OperationResponse operationResponse in dataServiceResponse)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.RequestResults.ElementAt(index), operationResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.RequestResults.ElementAt(index).HttpStatusCode, executionState.Cmd.RequestResults.ElementAt(index).ServiceRequestID, executionState.Cmd.RequestResults.ElementAt(index).ContentMd5, executionState.Cmd.RequestResults.ElementAt(index).Etag);
index++;
}
}
}
}
else if (queryResponse != null)
{
// Attempt to populate response headers
if (executionState.Req != null)
{
SetStorageCmdRequestResults(executionState.Cmd.CurrentResult, queryResponse);
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
}
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, ex.Message);
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
}
executionState.Result = cmd.ParseResponse(tempResult, executionState.Cmd.CurrentResult, cmd);
// clear exception
executionState.ExceptionRef = null;
}
TableExecutor.FinishRequestAttempt(executionState);
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
return(executionState.Result);
}
catch (Exception e)
{
TableExecutor.FinishRequestAttempt(executionState);
StorageException translatedException = ExecutorBase.TranslateDataServiceExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Cmd.ParseDataServiceError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
throw executionState.ExceptionRef;
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
if (delay > TimeSpan.Zero)
{
Thread.Sleep(delay);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
}while (shouldRetry);
// should never get here, either return, or throw;
throw new NotImplementedException(SR.InternalStorageError);
}
finally
{
ReleaseContext(cmd.Context);
}
}
private async static Task <T> ExecuteAsyncInternal <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
{
// Note all code below will reference state, not params directly, this will allow common code with multiple executors (APM, Sync, Async)
using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
{
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
// Create a new client
HttpClient client = cmd.BuildClient(cmd, executionState.OperationContext);
client.Timeout = TimeSpan.FromMilliseconds(int.MaxValue);
do
{
try
{
executionState.Init();
// 0. Begin Request
Executor.StartRequestAttempt(executionState);
// 1. Build request and content
executionState.CurrentOperation = ExecutorOperation.BeginOperation;
// Content is re-created every retry, as HttpClient disposes it after a successful request
HttpContent content = cmd.BuildContent != null?cmd.BuildContent(cmd, executionState.OperationContext) : null;
// This is so the old auth header etc is cleared out, the content is where serialization occurs which is the major perf hit
Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, cmd.Uri);
executionState.Req = cmd.BuildRequest(cmd, content, executionState.OperationContext);
// 2. Set Headers
Executor.ApplyUserHeaders(executionState);
// Let the user know we are ready to send
Executor.FireSendingRequest(executionState);
// 3. Sign Request is not needed, as HttpClient will call us
// 4. Set timeout
if (executionState.OperationExpiryTime.HasValue)
{
client.Timeout = executionState.RemainingTimeout;
}
Executor.CheckTimeout <T>(executionState, true);
}
catch (Exception ex)
{
Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);
// Store exception and throw here. All operations in this try would be non-retryable by default
StorageException storageEx = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
storageEx.IsRetryable = false;
executionState.ExceptionRef = storageEx;
#if WINDOWS_RT
// Need to throw wrapped Exception with message as serialized exception info stuff.
int hResult = WrappedStorageException.GenerateHResult(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
throw new WrappedStorageException(executionState.Cmd.CurrentResult.WriteAsXml(), executionState.ExceptionRef, hResult);
#else
throw executionState.ExceptionRef; // throw base exception for desktop
#endif
}
// Enter Retryable Section of execution
try
{
// Send Request
executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, token);
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
// Since HttpClient wont throw for non success, manually check and populate an exception
if (!executionState.Resp.IsSuccessStatusCode)
{
executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (cmd.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
// 8. (Potentially reads stream from server)
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
cmd.ResponseStream = await executionState.Resp.Content.ReadAsStreamAsync();
if (!cmd.RetrieveResponseStream)
{
cmd.DestinationStream = Stream.Null;
}
if (cmd.DestinationStream != null)
{
if (cmd.StreamCopyState == null)
{
cmd.StreamCopyState = new StreamDescriptor();
}
try
{
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, executionState, cmd.StreamCopyState, token);
}
finally
{
cmd.ResponseStream.Dispose();
cmd.ResponseStream = null;
}
}
// 9. Evaluate Response & Parse Results, (Stream potentially available here)
if (cmd.PostProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PostProcess;
Logger.LogInformational(executionState.OperationContext, SR.TracePostProcess);
executionState.Result = await cmd.PostProcessResponse(cmd, executionState.Resp, executionState.OperationContext);
}
executionState.CurrentOperation = ExecutorOperation.EndOperation;
Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
Executor.FinishRequestAttempt(executionState);
return(executionState.Result);
}
catch (Exception e)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
Executor.FinishRequestAttempt(executionState);
if (e is TaskCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
e = new TimeoutException(SR.TimeoutExceptionMessage, e);
}
StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
shouldRetry = false;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
}
finally
{
if (executionState.Resp != null)
{
executionState.Resp.Dispose();
executionState.Resp = null;
}
}
// potentially backoff
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
#if WINDOWS_RT
// Need to throw wrapped Exception with message as serialized exception info stuff.
int hResult = WrappedStorageException.GenerateHResult(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
throw new WrappedStorageException(executionState.Cmd.CurrentResult.WriteAsXml(), executionState.ExceptionRef, hResult);
#else
throw executionState.ExceptionRef; // throw base exception for desktop
#endif
}
else
{
if (cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
cmd.RecoveryAction(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay, token);
}
Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
}
}while (shouldRetry);
// should never get here
throw new NotImplementedException(SR.InternalStorageError);
}
}
private static void ConsiderRetry <T>(ExecutionState <T> executionState)
{
// Handle Retry
try
{
StorageException translatedException = TranslateExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, executionState.RestCMD.LocationMode);
Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);
IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
if (extendedRetryPolicy != null)
{
RetryContext retryContext = new RetryContext(
executionState.RetryCount++,
executionState.Cmd.CurrentResult,
executionState.CurrentLocation,
executionState.RestCMD.LocationMode);
RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
if (retryInfo != null)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
shouldRetry = true;
executionState.CurrentLocation = retryInfo.TargetLocation;
executionState.RestCMD.LocationMode = retryInfo.UpdatedLocationMode;
delay = retryInfo.RetryInterval;
}
}
else
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
}
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
executionState.UpdateCompletedSynchronously(false);
if (executionState.BackoffTimer == null)
{
executionState.BackoffTimer = new Timer(
Executor.RetryRequest <T>,
executionState,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
}
executionState.CancelDelegate = () =>
{
// Disabling the timer here, but there is still a scenario where the user calls cancel after
// the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
// if that happens, next retry will start and then stop immediately because of the cancelled flag.
Timer backoffTimer = executionState.BackoffTimer;
if (backoffTimer != null)
{
executionState.BackoffTimer = null;
backoffTimer.Dispose();
}
Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
Executor.CheckCancellation(executionState);
executionState.OnComplete();
};
}
else
{
// Start next request immediately
Executor.RetryRequest <T>(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
executionState.OnComplete();
}
}
private static void BeginGetRequestStream <T>(ExecutionState <T> executionState)
{
try
{
APMWithTimeout.RunWithTimeout(
executionState.Req.BeginGetRequestStream,
(getRequestStreamResp) =>
{
executionState.CompletedSynchronously = executionState.CompletedSynchronously && getRequestStreamResp.CompletedSynchronously;
try
{
executionState.ReqStream = executionState.Req.EndGetRequestStream(getRequestStreamResp);
// don't calculate md5 here as we should have already set this for auth purposes
executionState.RestCMD.SendStream.WriteToAsync(executionState.ReqStream, null /* maxLength */, executionState.OperationExpiryTime, false, executionState, executionState.OperationContext, null /* streamCopyState */, EndSendStreamCopy);
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.RequestCanceled)
{
// If the below condition is true, ExceptionRef is set anyway, so don't touch it
if (!Executor.CheckCancellation(executionState))
{
executionState.ExceptionRef = Exceptions.GenerateTimeoutException(executionState.Cmd.CurrentResult, null);
}
}
else
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
EndOperation(executionState);
}
catch (Exception ex)
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
EndOperation(executionState);
}
},
null,
executionState.RemainingTimeout,
(getRequestStreamResp) =>
{
try
{
executionState.ReqTimedOut = true;
executionState.Req.Abort();
}
catch (Exception)
{
// no op
}
});
}
catch (Exception ex)
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
EndOperation(executionState);
}
}
public static void InitRequest <T, INTERMEDIATE_TYPE>(ExecutionState <T> executionState)
{
try
{
executionState.Init();
// 0. Begin Request
TableExecutor.StartRequestAttempt(executionState);
if (TableExecutor.CheckTimeout <T>(executionState, false))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
lock (executionState.CancellationLockerObject)
{
if (TableExecutor.CheckCancellation(executionState))
{
TableExecutor.EndOperation <T, INTERMEDIATE_TYPE>(executionState);
return;
}
TableCommand <T, INTERMEDIATE_TYPE> tableCommandRef = executionState.Cmd as TableCommand <T, INTERMEDIATE_TYPE>;
// Execute Call
tableCommandRef.Begin(
(res) =>
{
executionState.UpdateCompletedSynchronously(res.CompletedSynchronously);
INTERMEDIATE_TYPE tResult = default(INTERMEDIATE_TYPE);
try
{
tResult = tableCommandRef.End(res);
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
// Attempt to populate response headers
if (executionState.Req != null)
{
executionState.Resp = GetResponseForRequest(executionState.Req);
FireResponseReceived(executionState);
}
}
catch (Exception ex)
{
lock (executionState.CancellationLockerObject)
{
if (executionState.CancelRequested)
{
// Ignore DSC exception if request was canceled.
return;
}
}
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
try
{
// Attempt to populate response headers
if (executionState.Req != null)
{
executionState.Resp = GetResponseForRequest(executionState.Req);
FireResponseReceived(executionState);
}
executionState.Result = tableCommandRef.ParseResponse(tResult, executionState.Cmd.CurrentResult, tableCommandRef);
// clear exception
executionState.ExceptionRef = null;
}
catch (Exception parseEx)
{
executionState.ExceptionRef = parseEx;
}
}
finally
{
EndOperation <T, INTERMEDIATE_TYPE>(executionState);
}
},
null);
if (tableCommandRef.Context != null)
{
executionState.CancelDelegate = tableCommandRef.Context.InternalCancel;
}
}
}
catch (Exception ex)
{
// Store exception and invoke callback here. All operations in this try would be non-retryable by default
if (executionState.ExceptionRef == null || !(executionState.ExceptionRef is StorageException))
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
executionState.OnComplete();
}
}
private static void BeginGetResponse <T>(ExecutionState <T> executionState)
{
try
{
APMWithTimeout.RunWithTimeout(
executionState.Req.BeginGetResponse,
(getRespRes) =>
{
try
{
executionState.CompletedSynchronously = executionState.CompletedSynchronously && getRespRes.CompletedSynchronously;
try
{
executionState.Resp = executionState.Req.EndGetResponse(getRespRes) as HttpWebResponse;
}
catch (WebException ex)
{
executionState.Resp = (HttpWebResponse)ex.Response;
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
}
FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (executionState.RestCMD.PreProcessResponse != null)
{
executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
}
CheckCancellation(executionState);
// 8. (Potentially reads stream from server)
if (executionState.RestCMD.RetrieveResponseStream && executionState.ExceptionRef == null)
{
executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();
if (executionState.RestCMD.DestinationStream != null)
{
if (executionState.RestCMD.StreamCopyState == null)
{
executionState.RestCMD.StreamCopyState = new StreamDescriptor();
}
executionState.RestCMD.ResponseStream.WriteToAsync(executionState.RestCMD.DestinationStream, null /* maxLength */, executionState.OperationExpiryTime, executionState.RestCMD.CalculateMd5ForResponseStream, executionState, executionState.OperationContext, executionState.RestCMD.StreamCopyState, EndResponseStreamCopy);
}
else
{
// Dont want to copy stream, just want to consume it so end
EndOperation(executionState);
}
}
else
{
// End
EndOperation(executionState);
}
}
catch (Exception ex)
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
EndOperation(executionState);
}
},
null,
executionState.RemainingTimeout,
(getRespRes) =>
{
try
{
executionState.ReqTimedOut = true;
executionState.Req.Abort();
}
catch (Exception)
{
// no op
}
});
}
catch (Exception ex)
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
EndOperation(executionState);
}
}
private static void EndGetResponse <T>(IAsyncResult getResponseResult)
{
ExecutionState <T> executionState = (ExecutionState <T>)getResponseResult.AsyncState;
executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
try
{
executionState.UpdateCompletedSynchronously(getResponseResult.CompletedSynchronously);
try
{
executionState.Resp = executionState.Req.EndGetResponse(getResponseResult) as HttpWebResponse;
}
catch (WebException ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
executionState.Resp = (HttpWebResponse)ex.Response;
if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
{
throw new TimeoutException();
}
if (executionState.Resp == null)
{
throw;
}
else
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
}
Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
Executor.FireResponseReceived(executionState);
// 7. Do Response parsing (headers etc, no stream available here)
if (executionState.RestCMD.PreProcessResponse != null)
{
executionState.CurrentOperation = ExecutorOperation.PreProcess;
executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);
// clear exception
executionState.ExceptionRef = null;
Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
}
Executor.CheckCancellation(executionState);
// 8. (Potentially reads stream from server)
if (executionState.ExceptionRef == null)
{
executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();
if (!executionState.RestCMD.RetrieveResponseStream)
{
executionState.RestCMD.DestinationStream = Stream.Null;
}
if (executionState.RestCMD.DestinationStream != null)
{
if (executionState.RestCMD.StreamCopyState == null)
{
executionState.RestCMD.StreamCopyState = new StreamDescriptor();
}
executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
executionState.RestCMD.ResponseStream.WriteToAsync(executionState.RestCMD.DestinationStream, null /* copyLength */, null /* maxLength */, executionState.RestCMD.CalculateMd5ForResponseStream, executionState, executionState.RestCMD.StreamCopyState, EndResponseStreamCopy);
}
else
{
// Dont want to copy stream, just want to consume it so end
Executor.EndOperation(executionState);
}
}
else
{
// End
Executor.EndOperation(executionState);
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TracePreProcessError, ex.Message);
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
Executor.EndOperation(executionState);
}
}
private static void EndOperation <T>(ExecutionState <T> executionState)
{
Executor.FinishRequestAttempt(executionState);
lock (executionState.CancellationLockerObject)
{
try
{
// If an operation has been canceled of timed out this should overwtie any exception
Executor.CheckCancellation(executionState);
Executor.CheckTimeout(executionState, true);
// Success
if (executionState.ExceptionRef == null)
{
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState, executionState.ExceptionRef);
executionState.OnComplete();
return;
}
}
catch (Exception ex)
{
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
finally
{
try
{
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
catch (Exception)
{
// no op
}
}
}
// Handle Retry
try
{
StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
TimeSpan delay = TimeSpan.FromMilliseconds(0);
bool shouldRetry = translatedException.IsRetryable &&
executionState.RetryPolicy != null?
executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext)
: false;
delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Timer backoffTimer = null;
backoffTimer = new Timer(
(obj) =>
{
executionState.CompletedSynchronously = false;
backoffTimer.Change(Timeout.Infinite, Timeout.Infinite);
backoffTimer.Dispose();
InitRequest(executionState);
},
null /* state */,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
// Start Next Request Immediately
InitRequest(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
executionState.OnComplete();
}
}
private static void EndOperation <T>(ExecutionState <T> executionState)
{
Executor.FinishRequestAttempt(executionState);
lock (executionState.CancellationLockerObject)
{
try
{
// If an operation has been canceled of timed out this should overwrite any exception
Executor.CheckCancellation(executionState);
Executor.CheckTimeout(executionState, true);
// Success
if (executionState.ExceptionRef == null)
{
// Step 9 - This will not be called if an exception is raised during stream copying
Executor.ProcessEndOfRequest(executionState);
executionState.OnComplete();
return;
}
}
catch (Exception ex)
{
Logger.LogWarning(executionState.OperationContext, SR.TracePostProcessError, ex.Message);
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
}
finally
{
try
{
if (executionState.ReqStream != null)
{
executionState.ReqStream.Dispose();
executionState.ReqStream = null;
}
if (executionState.Resp != null)
{
executionState.Resp.Close();
executionState.Resp = null;
}
}
catch (Exception)
{
// no op
}
}
}
// Handle Retry
try
{
StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
executionState.ExceptionRef = translatedException;
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);
bool shouldRetry = false;
TimeSpan delay = TimeSpan.Zero;
if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
{
shouldRetry = executionState.RetryPolicy.ShouldRetry(
executionState.RetryCount++,
executionState.Cmd.CurrentResult.HttpStatusCode,
executionState.ExceptionRef,
out delay,
executionState.OperationContext);
if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
{
delay = Constants.MaximumRetryBackoff;
}
}
if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
{
Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
// No Retry
executionState.OnComplete();
}
else
{
if (executionState.Cmd.RecoveryAction != null)
{
// I.E. Rewind stream etc.
executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
}
if (delay > TimeSpan.Zero)
{
Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
executionState.UpdateCompletedSynchronously(false);
if (executionState.BackoffTimer == null)
{
executionState.BackoffTimer = new Timer(
Executor.RetryRequest <T>,
executionState,
(int)delay.TotalMilliseconds,
Timeout.Infinite);
}
else
{
executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
}
executionState.CancelDelegate = () =>
{
// Disabling the timer here, but there is still a scenario where the user calls cancel after
// the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
// if that happens, next retry will start and then stop immediately because of the cancelled flag.
Timer backoffTimer = executionState.BackoffTimer;
if (backoffTimer != null)
{
executionState.BackoffTimer = null;
backoffTimer.Dispose();
}
Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
Executor.CheckCancellation(executionState);
executionState.OnComplete();
};
}
else
{
// Start next request immediately
Executor.RetryRequest <T>(executionState);
}
}
}
catch (Exception ex)
{
// Catch all ( i.e. users retry policy throws etc.)
Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
executionState.OnComplete();
}
}
protected static void FinishRequestAttempt <T>(ExecutionState <T> executionState)
{
executionState.Cmd.CurrentResult.EndTime = DateTime.Now;
executionState.OperationContext.EndTime = DateTime.Now;
}
private static void EndOperation <T>(ExecutionState <T> executionState)
{
EndOperationAsync <T>(executionState, false).Wait(); // Note that this will run sync if we pass in "false".
}
