Exemplo n.º 1
0
        public async static void AcquireAndReleaseLeaseTest(BlobContext context, string containerName, string blobName)
        {
            CloudStorageAccount account   = new CloudStorageAccount(new StorageCredentials(context.Account, context.Key), false);
            CloudBlobClient     client    = new CloudBlobClient(new Uri(context.Address), account.Credentials);
            CloudBlobContainer  container = client.GetContainerReference(containerName);
            CloudBlockBlob      blob      = container.GetBlockBlobReference(blobName);

            BlobTestBase.UploadText(blob, "Text sent to cloud", Encoding.UTF8);

            // acquire a release on the blob and check LeaseStatus to be "locked"
            OperationContext   opContext   = new OperationContext();
            HttpRequestMessage blobRequest = BlobHttpRequestMessageFactory.Lease(
                blob.Uri,
                context.Timeout,
                LeaseAction.Acquire,
                null /* proposed lease ID */,
                60 /* lease duration */,
                null /* break period */,
                null /* access condition */,
                null,
                opContext,
                SharedKeyCanonicalizer.Instance,
                context.Credentials);

            string leaseId = null;

            using (HttpResponseMessage response = await BlobTestUtils.GetResponse(blobRequest, context))
            {
                leaseId = HttpResponseParsers.GetHeader(response, "x-ms-lease-id");
                Assert.AreEqual <HttpStatusCode>(response.StatusCode, HttpStatusCode.Created);
            }

            blob.FetchAttributes();
            Assert.AreEqual <LeaseStatus>(blob.Properties.LeaseStatus, LeaseStatus.Locked);

            // release the release on the blob and check LeaseStatus to be "unlocked"
            opContext   = new OperationContext();
            blobRequest = BlobHttpRequestMessageFactory.Lease(
                blob.Uri,
                context.Timeout,
                LeaseAction.Release,
                null /* proposed lease ID */,
                null /* lease duration */,
                null /* break period */,
                AccessCondition.GenerateLeaseCondition(leaseId),
                null,
                opContext,
                SharedKeyCanonicalizer.Instance,
                context.Credentials);
            using (HttpResponseMessage response = await BlobTestUtils.GetResponse(blobRequest, context))
            {
                Assert.AreEqual <HttpStatusCode>(response.StatusCode, HttpStatusCode.OK);
            }

            blob.FetchAttributes();
            Assert.AreEqual <LeaseStatus>(blob.Properties.LeaseStatus, LeaseStatus.Unlocked);

            blob.Delete();
        }
Exemplo n.º 2
0
        /// <summary>
        /// Extracts the next visibility time from a response message header.
        /// </summary>
        /// <param name="response">The web response.</param>
        /// <returns>The time of next visibility stored in the header of the response.</returns>
        public static DateTime GetNextVisibleTime(HttpResponseMessage response)
        {
            CommonUtility.AssertNotNull("response", response);

            return(DateTime.Parse(
                       HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.NextVisibleTime),
                       System.Globalization.DateTimeFormatInfo.InvariantInfo,
                       System.Globalization.DateTimeStyles.AdjustToUniversal));
        }
Exemplo n.º 3
0
        /// <summary>
        /// Sets the SMB related file properties.
        /// </summary>
        /// <param name="response">The web response.</param>
        /// <param name="properties">The properties to modify.</param>
        public static void UpdateSmbProperties(HttpResponseMessage response, FileDirectoryProperties properties)
        {
            properties.filePermissionKey = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FilePermissionKey);
            properties.ntfsAttributes    = CloudFileNtfsAttributesHelper.ToAttributes(HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileAttributes));
            properties.creationTime      = DateTimeOffset.Parse(HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileCreationTime));
            properties.lastWriteTime     = DateTimeOffset.Parse(HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileLastWriteTime));
            properties.ChangeTime        = DateTimeOffset.Parse(HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileChangeTime));
            properties.DirectoryId       = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileId);
            properties.ParentId          = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.FileParentId);

            properties.filePermissionKeyToSet = null;
            properties.ntfsAttributesToSet    = null;
            properties.creationTimeToSet      = null;
            properties.lastWriteTimeToSet     = null;
        }
Exemplo n.º 4
0
        public static void BlobTypeHeader(HttpResponseMessage response, BlobType expectedValue)
        {
            Assert.IsNotNull(response);
            string   header = HttpResponseParsers.GetHeader(response, "x-ms-blob-type");
            BlobType?parsed = null;

            switch (header)
            {
            case "BlockBlob":
                parsed = BlobType.BlockBlob;
                break;

            case "PageBlob":
                parsed = BlobType.PageBlob;
                break;
            }
            Assert.IsNotNull(parsed);
            Assert.AreEqual(expectedValue, parsed);
        }
Exemplo n.º 5
0
        internal static void ValidateCPKHeaders(HttpResponseMessage response, BlobRequestOptions options, bool upload)
        {
            if (options?.CustomerProvidedKey == null)
            {
                return;
            }

            BlobCustomerProvidedKey key = options.CustomerProvidedKey;

            // Get ms-encryption-key-sha256 header from the response
            string encryptionKeySHA256Hash = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.ClientProvidedEncyptionKeyHash);

            if (!string.Equals(key.KeySHA256, encryptionKeySHA256Hash, StringComparison.OrdinalIgnoreCase))
            {
                throw new StorageException(SR.ClientProvidedKeyBadHash);
            }

            // If this is an upload
            if (upload)
            {
                // Get ms-request-server-encrypted header from the response
                string serverRequestEncrypted = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.ServerRequestEncrypted);

                // If header != "true"
                if (!string.Equals(Constants.HeaderConstants.TrueHeader, serverRequestEncrypted, StringComparison.OrdinalIgnoreCase))
                {
                    throw new StorageException(SR.ClientProvidedKeyEncryptionFailure);
                }
            }
            else
            {
                // Get ms-server-encrypted header
                string serviceEncrypted = HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.ServerEncrypted);

                // If header != "true"
                if (!string.Equals(Constants.HeaderConstants.TrueHeader, serviceEncrypted, StringComparison.OrdinalIgnoreCase))
                {
                    throw new StorageException(SR.ClientProvidedKeyEncryptionFailure);
                }
            }
        }
Exemplo n.º 6
0
 public static void RequestIdHeader(HttpResponseMessage response)
 {
     Assert.IsNotNull(response);
     Assert.IsNotNull(HttpResponseParsers.GetHeader(response, "x-ms-request-id"));
 }
Exemplo n.º 7
0
        public static async Task <T> ExecuteAsync <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
        {
            // Note all code below will reference state, not params directly, this will allow common code with async executor
            using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
                using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
                {
                    bool     shouldRetry = false;
                    TimeSpan delay       = TimeSpan.Zero;

                    // Create a new client
                    HttpClient client = cmd.HttpClient ?? HttpClientFactory.Instance;

                    do
                    {
                        try
                        {
                            executionState.Init();

                            // 0. Begin Request
                            Executor.StartRequestAttempt(executionState);

                            // Steps 1-4: Build Content/SetHeaders/
                            Executor.ProcessStartOfRequest(executionState, SR.TraceStartRequestAsync, timeoutTokenSource);
                            Executor.CheckTimeout <T>(executionState, true);
                        }
                        catch (Exception ex)
                        {
                            Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);

                            // Store exception and throw here. All operations in this try would be non-retryable by default. At this point, the request is not even made.
                            // Therefore, we will not get extended error info. Hence ParseError doesn't matter here.
                            StorageException storageEx = await ExecutorBase.TranslateExceptionBasedOnParseErrorAsync(ex, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, token).ConfigureAwait(false);

                            storageEx.IsRetryable       = false;
                            executionState.ExceptionRef = storageEx;
                            throw executionState.ExceptionRef;
                        }

                        // Enter Retryable Section of execution
                        try
                        {
                            // Send Request
                            executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);

                            executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutTokenSource.Token).ConfigureAwait(false);

                            executionState.CurrentOperation = ExecutorOperation.EndGetResponse;

                            // Check that echoed client ID matches the one we sent
                            var clientRequestId       = HttpRequestParsers.GetHeader(executionState.Req, Constants.HeaderConstants.ClientRequestIdHeader);
                            var echoedClientRequestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.ClientRequestIdHeader);

                            if (echoedClientRequestId != null && echoedClientRequestId != clientRequestId)
                            {
                                var requestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.RequestIdHeader);
                                var storageEx = new StorageException($"Echoed client request ID: {echoedClientRequestId} does not match sent client request ID: {clientRequestId}.  Service request ID: {requestId}")
                                {
                                    IsRetryable = false
                                };
                                executionState.ExceptionRef = storageEx;
                                throw storageEx;
                            }

                            // Since HttpClient wont throw for non success, manually check and populate an exception
                            if (!executionState.Resp.IsSuccessStatusCode)
                            {
                                // At this point, don't try to read the stream to parse the error
                                executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, token, executionState.Cmd.ParseErrorAsync).ConfigureAwait(false);
                            }

                            Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.ContentCrc64, executionState.Cmd.CurrentResult.Etag);
                            Executor.FireResponseReceived(executionState);


                            // 7. Do Response parsing (headers etc, no stream available here)
                            if (cmd.PreProcessResponse != null)
                            {
                                executionState.CurrentOperation = ExecutorOperation.PreProcess;

                                try
                                {
                                    executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                                    // clear exception
                                    executionState.ExceptionRef = null;
                                }
                                catch (Exception ex)
                                {
                                    executionState.ExceptionRef = ex;
                                }

                                Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
                            }

                            // 8. (Potentially reads stream from server)
                            executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
                            var responseStream = await executionState.Resp.Content.ReadAsStreamAsync().ConfigureAwait(false);

                            if (cmd.NetworkTimeout.HasValue)
                            {
                                responseStream = new TimeoutStream(responseStream, cmd.NetworkTimeout.Value);
                            }
                            cmd.ResponseStream = responseStream;

                            // The stream is now available in ResponseStream. Use the stream to parse out the response or error
                            if (executionState.ExceptionRef != null)
                            {
                                executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);

                                try
                                {
                                    cmd.ErrorStream = new MemoryStream();
                                    await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, ChecksumRequested.None, executionState, new StreamDescriptor(), timeoutTokenSource.Token).ConfigureAwait(false);

                                    cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
                                    executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream, executionState.Resp);
                                    throw executionState.ExceptionRef;
                                }
                                finally
                                {
                                    cmd.ResponseStream.Dispose();
                                    cmd.ResponseStream = null;

                                    cmd.ErrorStream.Dispose();
                                    cmd.ErrorStream = null;
                                }
                            }
                            else
                            {
                                if (!cmd.RetrieveResponseStream)
                                {
                                    cmd.DestinationStream = Stream.Null;
                                }

                                if (cmd.DestinationStream != null)
                                {
                                    if (cmd.StreamCopyState == null)
                                    {
                                        cmd.StreamCopyState = new StreamDescriptor();
                                    }

                                    try
                                    {
                                        executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
                                        await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, cmd.ChecksumRequestedForResponseStream, executionState, cmd.StreamCopyState, timeoutTokenSource.Token).ConfigureAwait(false);
                                    }
                                    finally
                                    {
                                        cmd.ResponseStream.Dispose();
                                        cmd.ResponseStream = null;
                                    }
                                }
                            }

                            await Executor.ProcessEndOfRequestAsync(executionState, token).ConfigureAwait(false);

                            Executor.FinishRequestAttempt(executionState);

                            return(executionState.Result);
                        }
                        catch (Exception e)
                        {
                            Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
                            Executor.FinishRequestAttempt(executionState);

                            if (e is OperationCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
                            {
                                e = new TimeoutException(SR.TimeoutExceptionMessage, e);
                            }

                            StorageException translatedException = await ExecutorBase.TranslateExceptionBasedOnParseErrorAsync(e, executionState.Cmd.CurrentResult, executionState.Resp, executionState.Cmd.ParseErrorAsync, token).ConfigureAwait(false);

                            executionState.ExceptionRef = translatedException;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                            shouldRetry = false;
                            if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                            {
                                executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
                                Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);

                                IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
                                if (extendedRetryPolicy != null)
                                {
                                    RetryContext retryContext = new RetryContext(
                                        executionState.RetryCount++,
                                        cmd.CurrentResult,
                                        executionState.CurrentLocation,
                                        cmd.LocationMode);

                                    RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
                                    if (retryInfo != null)
                                    {
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
                                        shouldRetry = true;
                                        executionState.CurrentLocation = retryInfo.TargetLocation;
                                        cmd.LocationMode = retryInfo.UpdatedLocationMode;
                                        delay            = retryInfo.RetryInterval;
                                    }
                                }
                                else
                                {
                                    shouldRetry = executionState.RetryPolicy.ShouldRetry(
                                        executionState.RetryCount++,
                                        cmd.CurrentResult.HttpStatusCode,
                                        executionState.ExceptionRef,
                                        out delay,
                                        executionState.OperationContext);
                                }

                                if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                                {
                                    delay = Constants.MaximumRetryBackoff;
                                }
                            }
                        }
                        finally
                        {
                            if (executionState.Resp != null)
                            {
                                executionState.Resp.Dispose();
                                executionState.Resp = null;
                            }
                        }

                        // potentially backoff
                        if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                        {
                            Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
                            throw executionState.ExceptionRef;
                        }
                        else
                        {
                            // I.E. Rewind stream etc.
                            cmd.RecoveryAction?.Invoke(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);

                            if (delay > TimeSpan.Zero)
                            {
                                await Task.Delay(delay, token).ConfigureAwait(false);
                            }

                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                        }

                        Executor.FireRetrying(executionState);
                    }while (shouldRetry);

                    // should never get here
                    throw new NotImplementedException(SR.InternalStorageError);
                }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Extracts the pop receipt from a web response header.
        /// </summary>
        /// <param name="response">The web response.</param>
        /// <returns>The pop receipt stored in the header of the response.</returns>
        public static string GetPopReceipt(HttpResponseMessage response)
        {
            CommonUtility.AssertNotNull("response", response);

            return(HttpResponseParsers.GetHeader(response, Constants.HeaderConstants.PopReceipt));
        }
Exemplo n.º 9
0
        private async static Task <T> ExecuteAsyncInternal <T>(RESTCommand <T> cmd, IRetryPolicy policy, OperationContext operationContext, CancellationToken token)
        {
            // Note all code below will reference state, not params directly, this will allow common code with multiple executors (APM, Sync, Async)
            using (ExecutionState <T> executionState = new ExecutionState <T>(cmd, policy, operationContext))
                using (CancellationTokenSource timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
                {
                    bool     shouldRetry = false;
                    TimeSpan delay       = TimeSpan.Zero;

                    // Note - The service accepts both api-version and x-ms-version and therefore it is ok to add x-ms-version to all requests.

                    // Create a new client
                    HttpClient client = cmd.HttpClient ?? HttpClientFactory.Instance;

                    do
                    {
                        try
                        {
                            executionState.Init();

                            // 0. Begin Request
                            Executor.StartRequestAttempt(executionState);

                            // 1. Build request and content
                            executionState.CurrentOperation = ExecutorOperation.BeginOperation;

                            // Content is re-created every retry, as HttpClient disposes it after a successful request
                            HttpContent content = cmd.BuildContent != null?cmd.BuildContent(cmd, executionState.OperationContext) : null;

                            // This is so the old auth header etc is cleared out, the content is where serialization occurs which is the major perf hit
                            Uri uri            = cmd.StorageUri.GetUri(executionState.CurrentLocation);
                            Uri transformedUri = cmd.Credentials.TransformUri(uri);
                            Logger.LogInformational(executionState.OperationContext, SR.TraceStartRequestAsync, transformedUri);
                            UriQueryBuilder builder = new UriQueryBuilder(executionState.RestCMD.Builder);
                            executionState.Req = cmd.BuildRequest(cmd, transformedUri, builder, content, cmd.ServerTimeoutInSeconds, executionState.OperationContext);

                            // 2. Set Headers
                            Executor.ApplyUserHeaders(executionState);

                            // Let the user know we are ready to send
                            Executor.FireSendingRequest(executionState);

                            // 3. Sign Request is not needed, as HttpClient will call us

                            // 4. Set timeout
                            if (executionState.OperationExpiryTime.HasValue)
                            {
                                // set the token to cancel after timing out, if the higher token hasn't already been cancelled
                                timeoutTokenSource.CancelAfter(executionState.RemainingTimeout);
                            }
                            else
                            {
                                // effectively prevent timeout
                                timeoutTokenSource.CancelAfter(int.MaxValue);
                            }

                            Executor.CheckTimeout <T>(executionState, true);
                        }
                        catch (Exception ex)
                        {
                            Logger.LogError(executionState.OperationContext, SR.TraceInitRequestError, ex.Message);

                            // Store exception and throw here. All operations in this try would be non-retryable by default
                            StorageException storageEx = await StorageException.TranslateExceptionAsync(ex, executionState.Cmd.CurrentResult, null, timeoutTokenSource.Token, executionState.Resp).ConfigureAwait(false);

                            storageEx.IsRetryable       = false;
                            executionState.ExceptionRef = storageEx;

                            throw executionState.ExceptionRef;
                        }

                        // Enter Retryable Section of execution
                        try
                        {
                            // Send Request
                            executionState.CurrentOperation = ExecutorOperation.BeginGetResponse;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceGetResponse);
                            executionState.Resp = await client.SendAsync(executionState.Req, HttpCompletionOption.ResponseHeadersRead, timeoutTokenSource.Token).ConfigureAwait(false);

                            executionState.CurrentOperation = ExecutorOperation.EndGetResponse;

                            // Check that echoed client ID matches the one we sent
                            var clientRequestId       = HttpRequestParsers.GetHeader(executionState.Req, Constants.HeaderConstants.ClientRequestIdHeader);
                            var echoedClientRequestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.ClientRequestIdHeader);

                            if (echoedClientRequestId != null && echoedClientRequestId != clientRequestId)
                            {
                                var requestId = HttpResponseParsers.GetHeader(executionState.Resp, Constants.HeaderConstants.RequestIdHeader);
                                var storageEx = new StorageException($"Echoed client request ID: {echoedClientRequestId} does not match sent client request ID: {clientRequestId}.  Service request ID: {requestId}")
                                {
                                    IsRetryable = false
                                };
                                executionState.ExceptionRef = storageEx;
                                throw storageEx;
                            }

                            // Since HttpClient wont throw for non success, manually check and populate an exception
                            if (!executionState.Resp.IsSuccessStatusCode)
                            {
                                // At this point, don't try to read the stream to parse the error
                                executionState.ExceptionRef = await Exceptions.PopulateStorageExceptionFromHttpResponseMessage(executionState.Resp, executionState.Cmd.CurrentResult, executionState.Cmd.ParseError).ConfigureAwait(false);
                            }

                            Logger.LogInformational(executionState.OperationContext, SR.TraceResponse, executionState.Cmd.CurrentResult.HttpStatusCode, executionState.Cmd.CurrentResult.ServiceRequestID, executionState.Cmd.CurrentResult.ContentMd5, executionState.Cmd.CurrentResult.ContentCrc64, executionState.Cmd.CurrentResult.Etag);
                            Executor.FireResponseReceived(executionState);

                            // 7. Do Response parsing (headers etc, no stream available here)
                            if (cmd.PreProcessResponse != null)
                            {
                                executionState.CurrentOperation = ExecutorOperation.PreProcess;

                                try
                                {
                                    executionState.Result = cmd.PreProcessResponse(cmd, executionState.Resp, executionState.ExceptionRef, executionState.OperationContext);

                                    // clear exception
                                    executionState.ExceptionRef = null;
                                }
                                catch (Exception ex)
                                {
                                    executionState.ExceptionRef = ex;
                                }

                                Logger.LogInformational(executionState.OperationContext, SR.TracePreProcessDone);
                            }

                            // 8. (Potentially reads stream from server)
                            executionState.CurrentOperation = ExecutorOperation.GetResponseStream;
                            var responseStream = await executionState.Resp.Content.ReadAsStreamAsync().ConfigureAwait(false);

                            if (cmd.NetworkTimeout.HasValue)
                            {
                                responseStream = new TimeoutStream(responseStream, cmd.NetworkTimeout.Value);
                            }
                            cmd.ResponseStream = responseStream;

                            // The stream is now available in ResponseStream. Use the stream to parse out the response or error
                            if (executionState.ExceptionRef != null)
                            {
                                executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                Logger.LogInformational(executionState.OperationContext, SR.TraceDownloadError);

                                try
                                {
                                    cmd.ErrorStream = new MemoryStream();
                                    await cmd.ResponseStream.WriteToAsync(cmd.ErrorStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, ChecksumRequested.None, executionState, new StreamDescriptor(), timeoutTokenSource.Token).ConfigureAwait(false);

                                    cmd.ErrorStream.Seek(0, SeekOrigin.Begin);
                                    executionState.ExceptionRef = StorageException.TranslateExceptionWithPreBufferedStream(executionState.ExceptionRef, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), cmd.ErrorStream, executionState.Resp);
                                    throw executionState.ExceptionRef;
                                }
                                finally
                                {
                                    cmd.ResponseStream.Dispose();
                                    cmd.ResponseStream = null;

                                    cmd.ErrorStream.Dispose();
                                    cmd.ErrorStream = null;
                                }
                            }
                            else
                            {
                                if (!cmd.RetrieveResponseStream)
                                {
                                    cmd.DestinationStream = Stream.Null;
                                }

                                if (cmd.DestinationStream != null)
                                {
                                    if (cmd.StreamCopyState == null)
                                    {
                                        cmd.StreamCopyState = new StreamDescriptor();
                                    }

                                    try
                                    {
                                        executionState.CurrentOperation = ExecutorOperation.BeginDownloadResponse;
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceDownload);
                                        await cmd.ResponseStream.WriteToAsync(cmd.DestinationStream, default(IBufferManager), null /* copyLength */, null /* maxLength */, cmd.ChecksumRequestedForResponseStream, executionState, cmd.StreamCopyState, timeoutTokenSource.Token).ConfigureAwait(false);
                                    }
                                    finally
                                    {
                                        cmd.ResponseStream.Dispose();
                                        cmd.ResponseStream = null;
                                    }
                                }
                            }

                            // 9. Evaluate Response & Parse Results, (Stream potentially available here)
                            if (cmd.PostProcessResponseAsync != null)
                            {
                                executionState.CurrentOperation = ExecutorOperation.PostProcess;
                                Logger.LogInformational(executionState.OperationContext, SR.TracePostProcess);
                                executionState.Result = await cmd.PostProcessResponseAsync(cmd, executionState.Resp, executionState.OperationContext, timeoutTokenSource.Token).ConfigureAwait(false);
                            }

                            executionState.CurrentOperation = ExecutorOperation.EndOperation;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceSuccess);
                            Executor.FinishRequestAttempt(executionState);

                            return(executionState.Result);
                        }
                        catch (Exception e)
                        {
                            Logger.LogWarning(executionState.OperationContext, SR.TraceGenericError, e.Message);
                            Executor.FinishRequestAttempt(executionState);

                            if (e is TaskCanceledException && (executionState.OperationExpiryTime.HasValue && DateTime.Now.CompareTo(executionState.OperationExpiryTime.Value) > 0))
                            {
                                e = new TimeoutException(SR.TimeoutExceptionMessage, e);
                            }
#if NETCORE || WINDOWS_RT
                            StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null), executionState.Resp);
#else
                            StorageException translatedException = StorageException.TranslateException(e, executionState.Cmd.CurrentResult, stream => executionState.Cmd.ParseError(stream, executionState.Resp, null));
#endif
                            executionState.ExceptionRef = translatedException;
                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetryCheck, executionState.RetryCount, executionState.Cmd.CurrentResult.HttpStatusCode, translatedException.IsRetryable ? "yes" : "no", translatedException.Message);

                            shouldRetry = false;
                            if (translatedException.IsRetryable && (executionState.RetryPolicy != null))
                            {
                                executionState.CurrentLocation = Executor.GetNextLocation(executionState.CurrentLocation, cmd.LocationMode);
                                Logger.LogInformational(executionState.OperationContext, SR.TraceNextLocation, executionState.CurrentLocation);

                                IExtendedRetryPolicy extendedRetryPolicy = executionState.RetryPolicy as IExtendedRetryPolicy;
                                if (extendedRetryPolicy != null)
                                {
                                    RetryContext retryContext = new RetryContext(
                                        executionState.RetryCount++,
                                        cmd.CurrentResult,
                                        executionState.CurrentLocation,
                                        cmd.LocationMode);

                                    RetryInfo retryInfo = extendedRetryPolicy.Evaluate(retryContext, executionState.OperationContext);
                                    if (retryInfo != null)
                                    {
                                        Logger.LogInformational(executionState.OperationContext, SR.TraceRetryInfo, retryInfo.TargetLocation, retryInfo.UpdatedLocationMode);
                                        shouldRetry = true;
                                        executionState.CurrentLocation = retryInfo.TargetLocation;
                                        cmd.LocationMode = retryInfo.UpdatedLocationMode;
                                        delay            = retryInfo.RetryInterval;
                                    }
                                }
                                else
                                {
                                    shouldRetry = executionState.RetryPolicy.ShouldRetry(
                                        executionState.RetryCount++,
                                        cmd.CurrentResult.HttpStatusCode,
                                        executionState.ExceptionRef,
                                        out delay,
                                        executionState.OperationContext);
                                }

                                if ((delay < TimeSpan.Zero) || (delay > Constants.MaximumRetryBackoff))
                                {
                                    delay = Constants.MaximumRetryBackoff;
                                }
                            }
                        }
                        finally
                        {
                            if (executionState.Resp != null)
                            {
                                executionState.Resp.Dispose();
                                executionState.Resp = null;
                            }
                        }

                        // potentially backoff
                        if (!shouldRetry || (executionState.OperationExpiryTime.HasValue && (DateTime.Now + delay).CompareTo(executionState.OperationExpiryTime.Value) > 0))
                        {
                            Logger.LogError(executionState.OperationContext, shouldRetry ? SR.TraceRetryDecisionTimeout : SR.TraceRetryDecisionPolicy, executionState.ExceptionRef.Message);
                            throw executionState.ExceptionRef;
                        }
                        else
                        {
                            if (cmd.RecoveryAction != null)
                            {
                                // I.E. Rewind stream etc.
                                cmd.RecoveryAction(cmd, executionState.Cmd.CurrentResult.Exception, executionState.OperationContext);
                            }

                            if (delay > TimeSpan.Zero)
                            {
                                await Task.Delay(delay, token).ConfigureAwait(false);
                            }

                            Logger.LogInformational(executionState.OperationContext, SR.TraceRetry);
                        }

                        Executor.FireRetrying(executionState);
                    }while (shouldRetry);

                    // should never get here
                    throw new NotImplementedException(SR.InternalStorageError);
                }
        }