Exemplo n.º 1
0
        public static T ExecuteSync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
            {
                bool     shouldRetry = false;
                TimeSpan delay       = TimeSpan.Zero;

                do
                {
                    try
                    {
                        executionState.Init();

                        // 0. Begin Request
                        Executor.StartRequestAttempt(executionState);

                        // Steps 1-4
                        Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestSync);

                        Executor.CheckTimeout <T>(executionState, true);
                    }
                    catch (Exception ex)
                    {
                        Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);

                        // Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made.
                        // Therefore, we will not get extended error info. Hence ParseError doesn't matter here.
                        StorageException storageEx = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);

                        storageEx.IsRetryable       = false;
                        executionState.ExceptionRef = storageEx;
                        throw executionState.ExceptionRef;
                    }

                    // Enter Retryable Section of execution
                    try
                    {
                        // 5. potentially upload data
                        if (cmd.SendStream != null)
                        {
                            executionState.CurrentOperation = ExecutorOperation.BeginGetRequestStream;
                            Logger.LogInformational(executionState.OperationContext, SR.TracePrepareUpload);
                            executionState.Req.Timeout = (int)executionState.RemainingTimeout.TotalMilliseconds;
                            executionState.ReqStream   = executionState.Req.GetRequestStream();

                            executionState.CurrentOperation = ExecutorOperation.BeginUploadRequest;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceUpload);
                            MultiBufferMemoryStream multiBufferMemoryStream = cmd.SendStream as MultiBufferMemoryStream;

                            try
                            {
                                if (multiBufferMemoryStream != null && !cmd.SendStreamLength.HasValue)
                                {
                                    multiBufferMemoryStream.FastCopyTo(executionState.ReqStream, executionState.OperationExpiryTime);
                                }
                                else
                                {
                                    // don't calculate md5 here as we should have already set this for auth purposes
                                    executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, cmd.SendStreamLength, null /* maxLength */, false, true, executionState, null /* streamCopyState */);
                                }

                                executionState.ReqStream.Flush();
                                executionState.ReqStream.Dispose();
                                executionState.ReqStream = null;
                            }
                            catch (Exception)
                            {
                                executionState.Req.Abort();
                                throw;
                            }
                        }

                        // 6. Get response
                        try
                        {
                            executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
                            executionState.Req.Timeout      = (int)executionState.RemainingTimeout.TotalMilliseconds;
                            executionState.Resp             = (HttpWebResponse)executionState.Req.GetResponse();
                            executionState.CurrentOperation = ExecutorOperation.EndGetResponse;
                        }
                        catch (WebException ex)
                        {
                            Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
                            executionState.Resp = (HttpWebResponse)ex.Response;

                            if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
                            {
                                throw new TimeoutException();
                            }

                            if (executionState.Resp == null)
                            {
                                throw;
                            }
                            else
                            {
                                executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
                            }
                        }

                        // Response
                        Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                        Executor.FireResponseReceived(executionState);

                        // 7. Do Response parsing (headers etc, no stream available here)
                        if (cmd.PreProcessResponse != null)
                        {
                            executionState.CurrentOperation = ExecutorOperation.PreProcess;
                            executionState.Result           = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                            // clear exception
                            executionState.ExceptionRef = null;
                            Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
                        }

                        // 8. (Potentially reads stream from server)
                        executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
                        cmd.ResponseStream = executionState.Resp.GetResponseStream();

                        if (!cmd.RetrieveResponseStream)
                        {
                            cmd.DestinationStream = Stream.Null;
                        }

                        if (cmd.DestinationStream != null)
                        {
                            if (cmd.StreamCopyState == null)
                            {
                                cmd.StreamCopyState = new StreamDescriptor();
                            }

                            try
                            {
                                executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
                                cmd.ResponseStream.WriteToSync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, false, executionState, cmd.StreamCopyState);
                            }
                            finally
                            {
                                cmd.ResponseStream.Dispose();
                                cmd.ResponseStream = null;
                            }
                        }

                        // Step 9 - This will not be called if an exception is raised during stream copying
                        Executor.ProcessEndOfRequest(executionState);

                        Executor.FinishRequestAttempt(executionState);
                        return(executionState.Result);
                    }
                    catch (Exception e)
                    {
                        Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
                        Executor.FinishRequestAttempt(executionState);

                        StorageException translatedException = ExecutorBase.TranslateExceptionBasedOnParseError(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);

                        executionState.ExceptionRef = translatedException;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                        shouldRetry = false;
                        if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                        {
                            executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
                            Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);

                            IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
                            if (extendedRetryPolicy != null)
                            {
                                RetryContext retryContext = new RetryContext(
                                    executionState.RetryCount++,
                                    cmd.CurrentResult,
                                    executionState.CurrentLocation,
                                    cmd.LocationMode);

                                RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
                                if (retryInfo != null)
                                {
                                    Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
                                    shouldRetry = true;
                                    executionState.CurrentLocation = retryInfo.TargetLocation;
                                    cmd.LocationMode = retryInfo.UpdatedLocationMode;
                                    delay            = retryInfo.RetryInterval;
                                }
                            }
                            else
                            {
                                shouldRetry = executionState.RetryPolicy.ShouldRetry(
                                    executionState.RetryCount++,
                                    cmd.CurrentResult.HttpStatusCode,
                                    executionState.ExceptionRef,
                                    out delay,
                                    executionState.OperationContext);
                            }

                            if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                            {
                                delay = Constants.MaximumRetryBackoff;
                            }
                        }
                    }
                    finally
                    {
                        if (executionState.Resp != null)
                        {
                            executionState.Resp.Close();
                            executionState.Resp = null;
                        }
                    }

                    if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                    {
                        Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
                        throw executionState.ExceptionRef;
                    }
                    else
                    {
                        if (executionState.Cmd.RecoveryAction != null)
                        {
                            // I.E. Rewind stream etc.
                            executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                        }

                        if (delay > TimeSpan.Zero)
                        {
                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);
                            Thread.Sleep(delay);
                        }

                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                    }

                    Executor.FireRetrying(executionState);
                }while (shouldRetry);
            }

            // should never get here, either return, or throw;
            throw new NotImplementedException(SR.InternalStorageError);
        }
Exemplo n.º 2
0
        private async static Task <T> ExecuteAsyncInternal <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
        {
            // Note all code below will reference state, not params directly, this will allow common code with multiple executors (APM, Sync, Async)
            using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
            {
                bool     shouldRetry = false;
                TimeSpan delay       = TimeSpan.Zero;

                // Create a new client
                HttpClient client = cmd.BuildClient(cmd, cmd.Handler, executionState.OperationContext);
                client.Timeout = TimeSpan.FromMilliseconds(int.MaxValue);

                do
                {
                    try
                    {
                        executionState.Init();

                        // 0. Begin Request
                        Executor.StartRequestAttempt(executionState);

                        // 1. Build request and content
                        executionState.CurrentOperation = ExecutorOperation.BeginOperation;

                        // Content is re-created every retry, as HttpClient disposes it after a successful request
                        HttpContent content = cmd.BuildContent != null?cmd.BuildContent(cmd, executionState.OperationContext) : null;

                        // This is so the old auth header etc is cleared out, the content is where serialization occurs which is the major perf hit
                        Uri uri            = cmd.StorageUri.GetUri(executionState.CurrentLocation);
                        Uri transformedUri = cmd.Credentials.TransformUri(uri);
                        Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, transformedUri);
                        UriQueryBuilder builder = new UriQueryBuilder(executionState.RestCMD.Builder);
                        executionState.Req = cmd.BuildRequest(cmd, transformedUri, builder, content, cmd.ServerTimeoutInSeconds, executionState.OperationContext);

                        // 2. Set Headers
                        Executor.ApplyUserHeaders(executionState);

                        // Let the user know we are ready to send
                        Executor.FireSendingRequest(executionState);

                        // 3. Sign Request is not needed, as HttpClient will call us

                        // 4. Set timeout
                        if (executionState.OperationExpiryTime.HasValue)
                        {
                            client.Timeout = executionState.RemainingTimeout;
                        }

                        Executor.CheckTimeout <T>(executionState, true);
                    }
                    catch (Exception ex)
                    {
                        Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);

                        // Store exception and throw here. All operations in this try would be non-retryable by default
                        StorageException storageEx = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult, null);
                        storageEx.IsRetryable       = false;
                        executionState.ExceptionRef = storageEx;

                        // Need to throw wrapped Exception with message as serialized exception info stuff.
                        int hResult = WrappedStorageException.GenerateHResult(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
                        throw new WrappedStorageException(executionState.Cmd.CurrentResult.WriteAsXml(), executionState.ExceptionRef, hResult);
                    }

                    // Enter Retryable Section of execution
                    try
                    {
                        // Send Request
                        executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
                        executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, token);

                        executionState.CurrentOperation = ExecutorOperation.EndGetResponse;

                        // Since HttpClient wont throw for non success, manually check and populate an exception
                        if (!executionState.Resp.IsSuccessStatusCode)
                        {
                            // At this point, don't try to read the stream to parse the error
                            executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult);
                        }

                        Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                        Executor.FireResponseReceived(executionState);

                        // 7. Do Response parsing (headers etc, no stream available here)
                        if (cmd.PreProcessResponse != null)
                        {
                            executionState.CurrentOperation = ExecutorOperation.PreProcess;

                            try
                            {
                                executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                                // clear exception
                                executionState.ExceptionRef = null;
                            }
                            catch (Exception ex)
                            {
                                executionState.ExceptionRef = ex;
                            }

                            Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
                        }

                        // 8. (Potentially reads stream from server)
                        executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
                        cmd.ResponseStream = await executionState.Resp.Content.ReadAsStreamAsync();

                        // The stream is now available in ResponseStream. Use the stream to parse out the response or error
                        if (executionState.ExceptionRef != null)
                        {
                            executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);

                            try
                            {
                                cmd.ErrorStream = new MemoryStream();
                                await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, null /* copyLength */, null /* maxLength */, false, executionState, new StreamDescriptor(), token);

                                cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
                                executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, null /* parseError */, cmd.ErrorStream);
                                throw executionState.ExceptionRef;
                            }
                            finally
                            {
                                cmd.ResponseStream.Dispose();
                                cmd.ResponseStream = null;

                                cmd.ErrorStream.Dispose();
                                cmd.ErrorStream = null;
                            }
                        }
                        else
                        {
                            if (!cmd.RetrieveResponseStream)
                            {
                                cmd.DestinationStream = Stream.Null;
                            }

                            if (cmd.DestinationStream != null)
                            {
                                if (cmd.StreamCopyState == null)
                                {
                                    cmd.StreamCopyState = new StreamDescriptor();
                                }

                                try
                                {
                                    executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                    Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
                                    await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, null /* copyLength */, null /* maxLength */, cmd.CalculateMd5ForResponseStream, executionState, cmd.StreamCopyState, token);
                                }
                                finally
                                {
                                    cmd.ResponseStream.Dispose();
                                    cmd.ResponseStream = null;
                                }
                            }
                        }

                        // 9. Evaluate Response & Parse Results, (Stream potentially available here)
                        if (cmd.PostProcessResponse != null)
                        {
                            executionState.CurrentOperation = ExecutorOperation.PostProcess;
                            Logger.LogInformational(executionState.OperationContext, SR.TracePostProcess);
                            executionState.Result = await cmd.PostProcessResponse(cmd, executionState.Resp, executionState.OperationContext);
                        }

                        executionState.CurrentOperation = ExecutorOperation.EndOperation;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
                        Executor.FinishRequestAttempt(executionState);

                        return(executionState.Result);
                    }
                    catch (Exception e)
                    {
                        Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
                        Executor.FinishRequestAttempt(executionState);

                        if (e is TaskCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
                        {
                            e = new TimeoutException(SR.TimeoutExceptionMessage, e);
                        }

                        StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, null);
                        executionState.ExceptionRef = translatedException;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                        shouldRetry = false;
                        if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                        {
                            executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
                            Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);

                            IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
                            if (extendedRetryPolicy != null)
                            {
                                RetryContext retryContext = new RetryContext(
                                    executionState.RetryCount++,
                                    cmd.CurrentResult,
                                    executionState.CurrentLocation,
                                    cmd.LocationMode);

                                RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
                                if (retryInfo != null)
                                {
                                    Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
                                    shouldRetry = true;
                                    executionState.CurrentLocation = retryInfo.TargetLocation;
                                    cmd.LocationMode = retryInfo.UpdatedLocationMode;
                                    delay            = retryInfo.RetryInterval;
                                }
                            }
                            else
                            {
                                shouldRetry = executionState.RetryPolicy.ShouldRetry(
                                    executionState.RetryCount++,
                                    cmd.CurrentResult.HttpStatusCode,
                                    executionState.ExceptionRef,
                                    out delay,
                                    executionState.OperationContext);
                            }

                            if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                            {
                                delay = Constants.MaximumRetryBackoff;
                            }
                        }
                    }
                    finally
                    {
                        if (executionState.Resp != null)
                        {
                            executionState.Resp.Dispose();
                            executionState.Resp = null;
                        }
                    }

                    // potentially backoff
                    if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                    {
                        Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
#if WINDOWS_RT
                        // Need to throw wrapped Exception with message as serialized exception info stuff.
                        int hResult = WrappedStorageException.GenerateHResult(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
                        throw new WrappedStorageException(executionState.Cmd.CurrentResult.WriteAsXml(), executionState.ExceptionRef, hResult);
#else
                        throw executionState.ExceptionRef; // throw base exception for desktop
#endif
                    }
                    else
                    {
                        if (cmd.RecoveryAction != null)
                        {
                            // I.E. Rewind stream etc.
                            cmd.RecoveryAction(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
                        }

                        if (delay > TimeSpan.Zero)
                        {
                            await Task.Delay(delay, token);
                        }

                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                    }
                }while (shouldRetry);

                // should never get here
                throw new NotImplementedException(SR.InternalStorageError);
            }
        }
Exemplo n.º 3
0
        private static void EndOperation <T>(ExecutionState <T> executionState)
        {
            Executor.FinishRequestAttempt(executionState);

            try
            {
                // If an operation has been canceled of timed out this should overwrite any exception
                Executor.CheckCancellation(executionState);
                Executor.CheckTimeout(executionState, true);

                // Success
                if (executionState.ExceptionRef == null)
                {
                    // Step 9 - This will not be called if an exception is raised during stream copying
                    Executor.ProcessEndOfRequest(executionState);
                    executionState.OnComplete();
                    return;
                }
            }
            catch (Exception ex)
            {
                Logger.LogWarning(executionState.OperationContext, SR.TracePostProcessError, ex.Message);
                executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
            }
            finally
            {
                try
                {
                    if (executionState.ReqStream != null)
                    {
                        executionState.ReqStream.Dispose();
                        executionState.ReqStream = null;
                    }

                    if (executionState.Resp != null)
                    {
                        executionState.Resp.Close();
                        executionState.Resp = null;
                    }
                }
                catch (Exception)
                {
                    // no op
                }
            }

            // Handle Retry
            try
            {
                StorageException translatedException = TranslateExceptionBasedOnParseError(executionState.ExceptionRef, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
                executionState.ExceptionRef = translatedException;
                Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                bool     shouldRetry = false;
                TimeSpan delay       = TimeSpan.Zero;
                if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                {
                    executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, executionState.RestCMD.LocationMode);
                    Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);

                    IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
                    if (extendedRetryPolicy != null)
                    {
                        RetryContext retryContext = new RetryContext(
                            executionState.RetryCount++,
                            executionState.Cmd.CurrentResult,
                            executionState.CurrentLocation,
                            executionState.RestCMD.LocationMode);

                        RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
                        if (retryInfo != null)
                        {
                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
                            shouldRetry = true;
                            executionState.CurrentLocation      = retryInfo.TargetLocation;
                            executionState.RestCMD.LocationMode = retryInfo.UpdatedLocationMode;
                            delay = retryInfo.RetryInterval;
                        }
                    }
                    else
                    {
                        shouldRetry = executionState.RetryPolicy.ShouldRetry(
                            executionState.RetryCount++,
                            executionState.Cmd.CurrentResult.HttpStatusCode,
                            executionState.ExceptionRef,
                            out delay,
                            executionState.OperationContext);
                    }

                    if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                    {
                        delay = Constants.MaximumRetryBackoff;
                    }
                }

                if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                {
                    Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);

                    // No Retry
                    executionState.OnComplete();
                }
                else
                {
                    if (executionState.Cmd.RecoveryAction != null)
                    {
                        // I.E. Rewind stream etc.
                        executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                    }

                    if (delay > TimeSpan.Zero)
                    {
                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryDelay, (int)delay.TotalMilliseconds);

                        executionState.UpdateCompletedSynchronously(false);
                        if (executionState.BackoffTimer == null)
                        {
                            executionState.BackoffTimer = new Timer(
                                Executor.RetryRequest <T>,
                                executionState,
                                (int)delay.TotalMilliseconds,
                                Timeout.Infinite);
                        }
                        else
                        {
                            executionState.BackoffTimer.Change((int)delay.TotalMilliseconds, Timeout.Infinite);
                        }

                        executionState.CancelDelegate = () =>
                        {
                            // Disabling the timer here, but there is still a scenario where the user calls cancel after
                            // the timer starts the next retry but before it sets the CancelDelegate back to null. However, even
                            // if that happens, next retry will start and then stop immediately because of the cancelled flag.
                            Timer backoffTimer = executionState.BackoffTimer;
                            if (backoffTimer != null)
                            {
                                executionState.BackoffTimer = null;
                                backoffTimer.Dispose();
                            }

                            Logger.LogWarning(executionState.OperationContext, SR.TraceAbortRetry);
                            Executor.CheckCancellation(executionState);
                            executionState.OnComplete();
                        };
                    }
                    else
                    {
                        // Start next request immediately
                        Executor.RetryRequest <T>(executionState);
                    }
                }
            }
            catch (Exception ex)
            {
                // Catch all ( i.e. users retry policy throws etc.)
                Logger.LogWarning(executionState.OperationContext, SR.TraceRetryError, ex.Message);
                executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
                executionState.OnComplete();
            }
        }
Exemplo n.º 4
0
        private static void EndGetResponse <T>(IAsyncResult getResponseResult)
        {
            ExecutionState <T> executionState = (ExecutionState <T>)getResponseResult.AsyncState;

            executionState.CurrentOperation = ExecutorOperation.EndGetResponse;

            try
            {
                executionState.UpdateCompletedSynchronously(getResponseResult.CompletedSynchronously);

                try
                {
                    executionState.Resp = executionState.Req.EndGetResponse(getResponseResult) as HttpWebResponse;
                }
                catch (WebException ex)
                {
                    Logger.LogWarning(executionState.OperationContext, SR.TraceGetResponseError, ex.Message);
                    executionState.Resp = (HttpWebResponse)ex.Response;

                    if (ex.Status == WebExceptionStatus.Timeout || executionState.ReqTimedOut)
                    {
                        throw new TimeoutException();
                    }

                    if (executionState.Resp == null)
                    {
                        throw;
                    }
                    else
                    {
                        // Store this exception for now. It will be parsed/thrown after the stream is read in step 8
                        executionState.ExceptionRef = ex;
                    }
                }

                Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.Etag);
                Executor.FireResponseReceived(executionState);

                // 7. Do Response parsing (headers etc, no stream available here)
                if (executionState.RestCMD.PreProcessResponse != null)
                {
                    executionState.CurrentOperation = ExecutorOperation.PreProcess;

                    try
                    {
                        executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                        // clear exception
                        executionState.ExceptionRef = null;
                        Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
                    }
                    catch (Exception ex)
                    {
                        executionState.ExceptionRef = ex;
                    }
                }

                Executor.CheckCancellation(executionState);

                executionState.CurrentOperation       = ExecutorOperation.GetResponseStream;
                executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();

                // 8. (Potentially reads stream from server)
                if (executionState.ExceptionRef != null)
                {
                    executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                    Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);

                    executionState.RestCMD.ErrorStream = new MemoryStream();
                    executionState.RestCMD.ResponseStream.WriteToAsync(executionState.RestCMD.ErrorStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, false /* calculateMd5 */, executionState, new StreamDescriptor(), EndResponseStreamCopy);
                }
                else
                {
                    if (!executionState.RestCMD.RetrieveResponseStream)
                    {
                        executionState.RestCMD.DestinationStream = Stream.Null;
                    }

                    if (executionState.RestCMD.DestinationStream != null)
                    {
                        if (executionState.RestCMD.StreamCopyState == null)
                        {
                            executionState.RestCMD.StreamCopyState = new StreamDescriptor();
                        }

                        executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                        Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
                        executionState.RestCMD.ResponseStream.WriteToAsync(executionState.RestCMD.DestinationStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, executionState.RestCMD.CalculateMd5ForResponseStream, executionState, executionState.RestCMD.StreamCopyState, EndResponseStreamCopy);
                    }
                    else
                    {
                        // Dont want to copy stream, just want to consume it so end
                        Executor.EndOperation(executionState);
                    }
                }
            }
            catch (Exception ex)
            {
                Logger.LogWarning(executionState.OperationContext, SR.TracePreProcessError, ex.Message);
                executionState.ExceptionRef = ExecutorBase.TranslateExceptionBasedOnParseError(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseError);
                Executor.EndOperation(executionState);
            }
        }
Exemplo n.º 5
0
        public static T ExecuteSync <T>(StorageCommandBase <T> cmd, IRetryPolicy policy, OperationContext operationContext)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext);
            bool shouldRetry = false;

            do
            {
                try
                {
                    executionState.Init();

                    // 0. Begin Request
                    Executor.StartRequestAttempt(executionState);

                    // Steps 1-4
                    Executor.ProcessStartOfRequest(executionState);

                    Executor.CheckTimeout <T>(executionState, true);

                    // Enter Retryable Section of execution
                    // 5. potentially upload data
                    if (executionState.RestCMD.SendStream != null)
                    {
                        // Reset timeout
                        executionState.Req.Timeout = executionState.RemainingTimeout;
                        executionState.ReqStream   = executionState.Req.GetRequestStream();
                        executionState.RestCMD.SendStream.WriteToSync(executionState.ReqStream, null /* maxLength */, executionState.OperationExpiryTime, false, true, executionState.OperationContext, null /* streamCopyState */); // don't calculate md5 here as we should have already set this for auth purposes
                        executionState.ReqStream.Flush();
                        executionState.ReqStream.Dispose();
                        executionState.ReqStream = null;
                    }

                    // 6. Get response
                    try
                    {
                        // Reset timeout
                        executionState.Req.Timeout = executionState.RemainingTimeout;
                        executionState.Resp        = (HttpWebResponse)executionState.Req.GetResponse();
                    }
                    catch (WebException ex)
                    {
                        executionState.Resp = (HttpWebResponse)ex.Response;
                        if (executionState.Resp == null)
                        {
                            throw;
                        }
                        else
                        {
                            executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                        }
                    }

                    // Response
                    Executor.FireResponseReceived(executionState);

                    // 7. Do Response parsing (headers etc, no stream available here)
                    if (executionState.RestCMD.PreProcessResponse != null)
                    {
                        executionState.Result = executionState.RestCMD.PreProcessResponse(executionState.RestCMD, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                        // clear exception
                        executionState.ExceptionRef = null;
                    }

                    // 8. (Potentially reads stream from server)
                    executionState.RestCMD.ResponseStream = executionState.Resp.GetResponseStream();

                    if (!executionState.RestCMD.RetrieveResponseStream)
                    {
                        executionState.RestCMD.DestinationStream = Stream.Null;
                    }

                    if (executionState.RestCMD.DestinationStream != null)
                    {
                        try
                        {
                            if (executionState.RestCMD.StreamCopyState == null)
                            {
                                executionState.RestCMD.StreamCopyState = new StreamDescriptor();
                            }

                            executionState.RestCMD.ResponseStream.WriteToSync(executionState.RestCMD.DestinationStream, null /* maxLength */, executionState.OperationExpiryTime, executionState.RestCMD.CalculateMd5ForResponseStream, false, executionState.OperationContext, executionState.RestCMD.StreamCopyState);
                        }
                        finally
                        {
                            executionState.RestCMD.ResponseStream.Dispose();
                            executionState.RestCMD.ResponseStream = null;
                        }
                    }

                    // Step 9 - This will not be called if an exception is raised during stream copying
                    Executor.ProcessEndOfRequest(executionState, executionState.ExceptionRef);

                    Executor.FinishRequestAttempt(executionState);

                    return(executionState.Result);
                }
                catch (Exception e)
                {
                    Executor.FinishRequestAttempt(executionState);

                    StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult);
                    executionState.ExceptionRef = translatedException;

                    TimeSpan delay = TimeSpan.FromMilliseconds(0);
                    shouldRetry = translatedException.IsRetryable &&
                                  executionState.RetryPolicy != null?
                                  executionState.RetryPolicy.ShouldRetry(
                        executionState.RetryCount++,
                        executionState.Cmd.CurrentResult.HttpStatusCode,
                        executionState.ExceptionRef,
                        out delay,
                        executionState.OperationContext)
                                      : false;

                    delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;

                    if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                    {
                        throw executionState.ExceptionRef;
                    }
                    else
                    {
                        if (executionState.Cmd.RecoveryAction != null)
                        {
                            // I.E. Rewind stream etc.
                            executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                        }

                        if (delay > TimeSpan.Zero)
                        {
                            Thread.Sleep(delay);
                        }
                    }
                }
                finally
                {
                    if (executionState.Resp != null)
                    {
                        executionState.Resp.Close();
                        executionState.Resp = null;
                    }
                }
            }while (shouldRetry);

            // should never get here, either return, or throw;
            throw new NotImplementedException(SR.InternalStorageError);
        }
Exemplo n.º 6
0
        private static void EndOperation <T>(ExecutionState <T> executionState)
        {
            Executor.FinishRequestAttempt(executionState);

            lock (executionState.CancellationLockerObject)
            {
                try
                {
                    // If an operation has been canceled of timed out this should overwtie any exception
                    Executor.CheckCancellation(executionState);
                    Executor.CheckTimeout(executionState, true);

                    // Success
                    if (executionState.ExceptionRef == null)
                    {
                        // Step 9 - This will not be called if an exception is raised during stream copying
                        Executor.ProcessEndOfRequest(executionState, executionState.ExceptionRef);
                        executionState.OnComplete();
                        return;
                    }
                }
                catch (Exception ex)
                {
                    executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                }
                finally
                {
                    try
                    {
                        if (executionState.Resp != null)
                        {
                            executionState.Resp.Close();
                            executionState.Resp = null;
                        }
                    }
                    catch (Exception)
                    {
                        // no op
                    }
                }
            }

            // Handle Retry
            try
            {
                StorageException translatedException = StorageException.TranslateException(executionState.ExceptionRef, executionState.Cmd.CurrentResult);
                executionState.ExceptionRef = translatedException;

                TimeSpan delay       = TimeSpan.FromMilliseconds(0);
                bool     shouldRetry = translatedException.IsRetryable &&
                                       executionState.RetryPolicy != null?
                                       executionState.RetryPolicy.ShouldRetry(
                    executionState.RetryCount++,
                    executionState.Cmd.CurrentResult.HttpStatusCode,
                    executionState.ExceptionRef,
                    out delay,
                    executionState.OperationContext)
                                           : false;

                delay = delay.TotalMilliseconds <0 || delay> Constants.MaximumRetryBackoff ? Constants.MaximumRetryBackoff : delay;

                if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                {
                    // No Retry
                    executionState.OnComplete();
                }
                else
                {
                    if (executionState.Cmd.RecoveryAction != null)
                    {
                        // I.E. Rewind stream etc.
                        executionState.Cmd.RecoveryAction(executionState.Cmd, executionState.ExceptionRef, executionState.OperationContext);
                    }

                    if (delay > TimeSpan.Zero)
                    {
                        Timer backoffTimer = null;

                        backoffTimer = new Timer(
                            (obj) =>
                        {
                            executionState.UpdateCompletedSynchronously(false);
                            backoffTimer.Change(Timeout.Infinite, Timeout.Infinite);
                            backoffTimer.Dispose();
                            InitRequest(executionState);
                        },
                            null /* state */,
                            (int)delay.TotalMilliseconds,
                            Timeout.Infinite);
                    }
                    else
                    {
                        // Start Next Request Immediately
                        InitRequest(executionState);
                    }
                }
            }
            catch (Exception ex)
            {
                // Catch all ( i.e. users retry policy throws etc.)
                executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                executionState.OnComplete();
            }
        }
Exemplo n.º 7
0
        private static void BeginGetRequestStream <T>(ExecutionState <T> executionState)
        {
            try
            {
                APMWithTimeout.RunWithTimeout(
                    executionState.Req.BeginGetRequestStream,
                    (getRequestStreamResp) =>
                {
                    executionState.UpdateCompletedSynchronously(getRequestStreamResp.CompletedSynchronously);

                    try
                    {
                        executionState.ReqStream = executionState.Req.EndGetRequestStream(getRequestStreamResp);

                        // don't calculate md5 here as we should have already set this for auth purposes
                        executionState.RestCMD.SendStream.WriteToAsync(executionState.ReqStream, null /* maxLength */, executionState.OperationExpiryTime, false, executionState, executionState.OperationContext, null /* streamCopyState */, EndSendStreamCopy);
                    }
                    catch (WebException ex)
                    {
                        if (ex.Status == WebExceptionStatus.RequestCanceled)
                        {
                            // If the below condition is true, ExceptionRef is set anyway, so don't touch it
                            if (!Executor.CheckCancellation(executionState))
                            {
                                executionState.ExceptionRef = Exceptions.GenerateTimeoutException(executionState.Cmd.CurrentResult, null);
                            }
                        }
                        else
                        {
                            executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                        }

                        EndOperation(executionState);
                    }
                    catch (Exception ex)
                    {
                        executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                        EndOperation(executionState);
                    }
                },
                    null,
                    executionState.RemainingTimeout,
                    (getRequestStreamResp) =>
                {
                    try
                    {
                        executionState.ReqTimedOut = true;
                        executionState.Req.Abort();
                    }
                    catch (Exception)
                    {
                        // no op
                    }
                });
            }
            catch (Exception ex)
            {
                executionState.ExceptionRef = StorageException.TranslateException(ex, executionState.Cmd.CurrentResult);
                EndOperation(executionState);
            }
        }