Esempio n. 1
0
        public async Task MoveNext_TokenCanceledDuringCall_ThrowError()
        {
            // Arrange
            var cts = new CancellationTokenSource();

            var httpClient = ClientTestHelpers.CreateTestClient(request =>
            {
                var stream  = new SyncPointMemoryStream();
                var content = new StreamContent(stream);
                return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content)));
            });

            var channel = GrpcChannel.ForAddress(httpClient.BaseAddress, new GrpcChannelOptions {
                HttpClient = httpClient
            });
            var call = new GrpcCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, new CallOptions(), channel);

            call.StartServerStreaming(new HelloRequest());

            // Act
            var moveNextTask1 = call.ClientStreamReader !.MoveNext(cts.Token);

            // Assert
            Assert.IsFalse(moveNextTask1.IsCompleted);

            cts.Cancel();

            var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => moveNextTask1).DefaultTimeout();

            Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
        }
Esempio n. 2
0
 public WinHttpUnaryContent(TRequest request, Func <TRequest, Stream, ValueTask> startCallback, GrpcCall <TRequest, TResponse> call)
 {
     _request            = request;
     _startCallback      = startCallback;
     _call               = call;
     Headers.ContentType = GrpcProtocolConstants.GrpcContentTypeHeaderValue;
 }
Esempio n. 3
0
        public async Task MoveNext_MultipleCallsWithoutAwait_ThrowError()
        {
            // Arrange
            var httpClient = ClientTestHelpers.CreateTestClient(request =>
            {
                var stream  = new SyncPointMemoryStream();
                var content = new StreamContent(stream);
                return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content)));
            });

            var channel = GrpcChannel.ForAddress(httpClient.BaseAddress, new GrpcChannelOptions {
                HttpClient = httpClient
            });
            var call = new GrpcCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, new CallOptions(), channel);

            call.StartServerStreaming(new HelloRequest());

            // Act
            var moveNextTask1 = call.ClientStreamReader !.MoveNext(CancellationToken.None);
            var moveNextTask2 = call.ClientStreamReader.MoveNext(CancellationToken.None);

            // Assert
            Assert.IsFalse(moveNextTask1.IsCompleted);

            var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => moveNextTask2).DefaultTimeout();

            Assert.AreEqual("Can't read the next message because the previous read is still in progress.", ex.Message);
        }
Esempio n. 4
0
 static async ValueTask WriteBufferedMessages(GrpcCall <TRequest, TResponse> call, Stream requestStream, ReadOnlyMemory <byte>[] bufferedMessages)
 {
     foreach (var writtenMessage in bufferedMessages)
     {
         await call.WriteMessageAsync(requestStream, writtenMessage, call.CancellationToken).ConfigureAwait(false);
     }
 }
Esempio n. 5
0
        public void MoveNext_TokenCanceledDuringCall_ThrowError()
        {
            // Arrange
            var cts = new CancellationTokenSource();

            var httpClient = TestHelpers.CreateTestClient(request =>
            {
                var stream  = new SyncPointMemoryStream();
                var content = new StreamContent(stream);
                return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content)));
            });

            var call = new GrpcCall <HelloRequest, HelloReply>(TestHelpers.ServiceMethod, new CallOptions(), SystemClock.Instance, NullLoggerFactory.Instance);

            call.StartServerStreaming(httpClient, new HelloRequest());

            // Act
            var moveNextTask1 = call.ClientStreamReader !.MoveNext(cts.Token);

            // Assert
            Assert.IsFalse(moveNextTask1.IsCompleted);

            cts.Cancel();

            var ex = Assert.ThrowsAsync <RpcException>(async() => await moveNextTask1.DefaultTimeout());

            Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
        }
Esempio n. 6
0
        public void MoveNext_MultipleCallsWithoutAwait_ThrowError()
        {
            // Arrange
            var httpClient = TestHelpers.CreateTestClient(request =>
            {
                var stream  = new SyncPointMemoryStream();
                var content = new StreamContent(stream);
                return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content)));
            });

            var call = new GrpcCall <HelloRequest, HelloReply>(TestHelpers.ServiceMethod, new CallOptions(), SystemClock.Instance, NullLoggerFactory.Instance);

            call.StartServerStreaming(httpClient, new HelloRequest());

            // Act
            var moveNextTask1 = call.ClientStreamReader !.MoveNext(CancellationToken.None);
            var moveNextTask2 = call.ClientStreamReader.MoveNext(CancellationToken.None);

            // Assert
            Assert.IsFalse(moveNextTask1.IsCompleted);

            var ex = Assert.ThrowsAsync <InvalidOperationException>(async() => await moveNextTask2.DefaultTimeout());

            Assert.AreEqual("Cannot read next message because the previous read is in progress.", ex.Message);
        }
Esempio n. 7
0
        private static bool IsSuccessfulStreamingCall(Status responseStatus, GrpcCall <TRequest, TResponse> call)
        {
            if (responseStatus.StatusCode != StatusCode.OK)
            {
                return(false);
            }

            return(call.Method.Type == MethodType.ServerStreaming || call.Method.Type == MethodType.DuplexStreaming);
        }
        public async Task ReportsNewLeader(GrpcCall call)
        {
            EndPoint actual = default;
            var      sut    = new ReportLeaderInterceptor(ex => actual = ex);

            var result = await Assert.ThrowsAsync <NotLeaderException>(() =>
                                                                       call(sut, Task.FromException <object>(new NotLeaderException("a.host", 2112))));

            Assert.Equal(result.LeaderEndpoint, actual);
        }
Esempio n. 9
0
        private HttpContent CreatePushUnaryContent(TRequest request, GrpcCall <TRequest, TResponse> call)
        {
            return(!Channel.IsWinHttp
                ? new PushUnaryContent <TRequest, TResponse>(request, WriteAsync)
                : new WinHttpUnaryContent <TRequest, TResponse>(request, WriteAsync, call));

            ValueTask WriteAsync(TRequest request, Stream stream)
            {
                return(WriteNewMessage(call, stream, call.Options, request));
            }
        }
Esempio n. 10
0
        private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>(
            Method <TRequest, TResponse> method,
            CallOptions options)
            where TRequest : class
            where TResponse : class
        {
            if (_client.BaseAddress == null)
            {
                throw new InvalidOperationException("Unable to send the gRPC call because no server address has been configured. " +
                                                    "Set HttpClient.BaseAddress on the HttpClient used to created to gRPC client.");
            }

            var call = new GrpcCall <TRequest, TResponse>(method, options, this);

            return(call);
        }
        private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>(
            Method <TRequest, TResponse> method,
            CallOptions options,
            out Action disposeAction)
            where TRequest : class
            where TResponse : class
        {
            if (_client.BaseAddress == null)
            {
                throw new InvalidOperationException("Unable to send the gRPC call because no server address has been configured. " +
                                                    "Set HttpClient.BaseAddress on the HttpClient used to created to gRPC client.");
            }

            CancellationTokenSource?linkedCts = null;

            // Use propagated deadline if it is small than the specified deadline
            if (Deadline < options.Deadline)
            {
                options = options.WithDeadline(Deadline);
            }

            if (CancellationToken.CanBeCanceled)
            {
                if (options.CancellationToken.CanBeCanceled)
                {
                    // If both propagated and options cancellation token can be canceled
                    // then set a new linked token of both
                    linkedCts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, options.CancellationToken);
                    options   = options.WithCancellationToken(linkedCts.Token);
                }
                else
                {
                    options = options.WithCancellationToken(CancellationToken);
                }
            }

            var call = new GrpcCall <TRequest, TResponse>(method, options, Clock, _loggerFactory);

            // Clean up linked cancellation token
            disposeAction = linkedCts != null
                ? () => { call.Dispose(); linkedCts !.Dispose(); }
                : (Action)call.Dispose;

            return(call);
        }
Esempio n. 12
0
        public static async Task <TResponse?> ReadMessageAsync <TResponse>(
#endif
            this Stream responseStream,
            GrpcCall call,
            Func <DeserializationContext, TResponse> deserializer,
            string grpcEncoding,
            bool singleMessage,
            CancellationToken cancellationToken)
            where TResponse : class
        {
            byte[]? buffer = null;

            try
            {
                GrpcCallLog.ReadingMessage(call.Logger);
                cancellationToken.ThrowIfCancellationRequested();

                // Buffer is used to read header, then message content.
                // This size was randomly chosen to hopefully be big enough for many small messages.
                // If the message is larger then the array will be replaced when the message size is known.
                buffer = ArrayPool <byte> .Shared.Rent(minimumLength : 4096);

                int read;
                var received = 0;
                while ((read = await responseStream.ReadAsync(buffer.AsMemory(received, GrpcProtocolConstants.HeaderSize - received), cancellationToken).ConfigureAwait(false)) > 0)
                {
                    received += read;

                    if (received == GrpcProtocolConstants.HeaderSize)
                    {
                        break;
                    }
                }

                if (received < GrpcProtocolConstants.HeaderSize)
                {
                    if (received == 0)
                    {
                        GrpcCallLog.NoMessageReturned(call.Logger);
                        return(default);
Esempio n. 13
0
        private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>(
            Method <TRequest, TResponse> method,
            CallOptions options,
            out Action disposeAction)
            where TRequest : class
            where TResponse : class
        {
            CancellationTokenSource?linkedCts = null;

            // Use propagated deadline if it is small than the specified deadline
            if (Deadline < options.Deadline)
            {
                options = options.WithDeadline(Deadline);
            }

            if (CancellationToken.CanBeCanceled)
            {
                if (options.CancellationToken.CanBeCanceled)
                {
                    // If both propagated and options cancellation token can be canceled
                    // then set a new linked token of both
                    linkedCts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, options.CancellationToken);
                    options   = options.WithCancellationToken(linkedCts.Token);
                }
                else
                {
                    options = options.WithCancellationToken(CancellationToken);
                }
            }

            var call = new GrpcCall <TRequest, TResponse>(method, options, Clock, _loggerFactory);

            // Clean up linked cancellation token
            disposeAction = linkedCts != null
                ? () => { call.Dispose(); linkedCts !.Dispose(); }
                : (Action)call.Dispose;

            return(call);
        }
Esempio n. 14
0
        private PushStreamContent <TRequest, TResponse> CreatePushStreamContent(GrpcCall <TRequest, TResponse> call, HttpContentClientStreamWriter <TRequest, TResponse> clientStreamWriter)
        {
            return(new PushStreamContent <TRequest, TResponse>(clientStreamWriter, async requestStream =>
            {
                ValueTask writeTask;
                lock (Lock)
                {
                    Log.SendingBufferedMessages(Logger, BufferedMessages.Count);

                    if (BufferedMessages.Count == 0)
                    {
#if NETSTANDARD2_0
                        writeTask = Task.CompletedTask;
#else
                        writeTask = default;
#endif
                    }
                    else if (BufferedMessages.Count == 1)
                    {
                        writeTask = call.WriteMessageAsync(requestStream, BufferedMessages[0], call.CancellationToken);
                    }
                    else
                    {
                        // Copy messages to a new collection in lock for thread-safety.
                        var bufferedMessageCopy = BufferedMessages.ToArray();
                        writeTask = WriteBufferedMessages(call, requestStream, bufferedMessageCopy);
                    }
                }

                await writeTask.ConfigureAwait(false);

                if (ClientStreamComplete)
                {
                    await call.ClientStreamWriter !.CompleteAsync().ConfigureAwait(false);
                }
            }));
Esempio n. 15
0
        private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
        {
            GrpcCall <TRequest, TResponse> call;

            lock (Lock)
            {
                if (CommitedCallTask.IsCompletedSuccessfully())
                {
                    // Call has already been commited. This could happen if written messages exceed
                    // buffer limits, which causes the call to immediately become commited and to clear buffers.
                    return;
                }

                OnStartingAttempt();

                call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
                _activeCalls.Add(call);

                startCallFunc(call);

                SetNewActiveCallUnsynchronized(call);
            }

            Status?responseStatus;

            HttpResponseMessage?httpResponse = null;

            try
            {
                call.CancellationToken.ThrowIfCancellationRequested();

                CompatibilityExtensions.Assert(call._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled.");
                httpResponse = await call._httpResponseTask.ConfigureAwait(false);

                responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
            }
            catch (Exception ex)
            {
                call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
            }

            if (CancellationTokenSource.IsCancellationRequested)
            {
                CommitCall(call, CommitReason.Canceled);
                return;
            }

            // Check to see the response returned from the server makes the call commited
            // Null status code indicates the headers were valid and a "Response-Headers" response
            // was received from the server.
            // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
            if (responseStatus == null)
            {
                // Headers were returned. We're commited.
                CommitCall(call, CommitReason.ResponseHeadersReceived);

                // Wait until the call has finished and then check its status code
                // to update retry throttling tokens.
                var status = await call.CallTask.ConfigureAwait(false);

                if (status.StatusCode == StatusCode.OK)
                {
                    RetryAttemptCallSuccess();
                }
            }
            else
            {
                var status = responseStatus.Value;

                var retryPushbackMS = GetRetryPushback(httpResponse);

                if (retryPushbackMS < 0)
                {
                    RetryAttemptCallFailure();
                }
                else if (_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
                {
                    // Needs to happen before interrupt.
                    RetryAttemptCallFailure();

                    // No need to interrupt if we started with no delay and all calls
                    // have already been made when hedging starting.
                    if (_delayInterruptTcs != null)
                    {
                        lock (Lock)
                        {
                            if (retryPushbackMS >= 0)
                            {
                                _pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
                            }
                            _delayInterruptTcs.TrySetResult(null);
                        }
                    }
                }
                else
                {
                    CommitCall(call, CommitReason.FatalStatusCode);
                }
            }

            lock (Lock)
            {
                if (IsDeadlineExceeded())
                {
                    // Deadline has been exceeded so immediately commit call.
                    CommitCall(call, CommitReason.DeadlineExceeded);
                }
                else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
                {
                    // This is the last active call and no more will be made.
                    CommitCall(call, CommitReason.ExceededAttemptCount);
                }
                else if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
                {
                    // This is the last active call and throttling is active.
                    CommitCall(call, CommitReason.Throttled);
                }
                else
                {
                    // Call isn't used and can be cancelled.
                    // Note that the call could have already been removed and disposed if the
                    // hedging call has been finalized or disposed.
                    if (_activeCalls.Remove(call))
                    {
                        call.Dispose();
                    }
                }
            }
        }
Esempio n. 16
0
 public LengthUnaryContent(TRequest content, GrpcCall <TRequest, TResponse> call, MediaTypeHeaderValue mediaType)
 {
     _content            = content;
     _call               = call;
     Headers.ContentType = mediaType;
 }
Esempio n. 17
0
 protected override void OnCommitCall(IGrpcCall <TRequest, TResponse> call)
 {
     _activeCall = null;
 }
Esempio n. 18
0
        private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
        {
            Log.StartingRetryWorker(Logger);

            try
            {
                // This is the main retry loop. It will:
                // 1. Check the result of the active call was successful.
                // 2. If it was unsuccessful then evaluate if the call can be retried.
                // 3. If it can be retried then start a new active call and begin again.
                while (true)
                {
                    GrpcCall <TRequest, TResponse> currentCall;
                    lock (Lock)
                    {
                        // Start new call.
                        OnStartingAttempt();

                        currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
                        startCallFunc(currentCall);

                        SetNewActiveCallUnsynchronized(currentCall);

                        if (CommitedCallTask.IsCompletedSuccessfully())
                        {
                            // Call has already been commited. This could happen if written messages exceed
                            // buffer limits, which causes the call to immediately become commited and to clear buffers.
                            return;
                        }
                    }

                    Status?responseStatus;

                    HttpResponseMessage?httpResponse = null;
                    try
                    {
                        httpResponse = await currentCall.HttpResponseTask.ConfigureAwait(false);

                        responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
                    }
                    catch (RpcException ex)
                    {
                        // A "drop" result from the load balancer should immediately stop the call,
                        // including ignoring the retry policy.
                        var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
                        if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
                        {
                            CommitCall(currentCall, CommitReason.Drop);
                            return;
                        }

                        currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
                    }
                    catch (Exception ex)
                    {
                        currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
                    }

                    CancellationTokenSource.Token.ThrowIfCancellationRequested();

                    // Check to see the response returned from the server makes the call commited.
                    // 1. Null status code indicates the headers were valid and a "Response-Headers" response
                    //    was received from the server.
                    // 2. An OK response status at this point means a streaming call completed without
                    //    sending any messages to the client.
                    //
                    // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
                    if (responseStatus == null)
                    {
                        // Headers were returned. We're commited.
                        CommitCall(currentCall, CommitReason.ResponseHeadersReceived);

                        responseStatus = await currentCall.CallTask.ConfigureAwait(false);

                        if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK)
                        {
                            RetryAttemptCallSuccess();
                        }

                        // Commited so exit retry loop.
                        return;
                    }
                    else if (IsSuccessfulStreamingCall(responseStatus.GetValueOrDefault(), currentCall))
                    {
                        // Headers were returned. We're commited.
                        CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
                        RetryAttemptCallSuccess();

                        // Commited so exit retry loop.
                        return;
                    }

                    if (CommitedCallTask.IsCompletedSuccessfully())
                    {
                        // Call has already been commited. This could happen if written messages exceed
                        // buffer limits, which causes the call to immediately become commited and to clear buffers.
                        return;
                    }

                    var status          = responseStatus.GetValueOrDefault();
                    var retryPushbackMS = GetRetryPushback(httpResponse);

                    // Failures only count towards retry throttling if they have a known, retriable status.
                    // This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling.
                    if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) ||
                        retryPushbackMS < 0)
                    {
                        RetryAttemptCallFailure();
                    }

                    var result = EvaluateRetry(status, retryPushbackMS);
                    Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null);

                    if (result == null)
                    {
                        TimeSpan delayDuration;
                        if (retryPushbackMS != null)
                        {
                            delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
                            _nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault();
                        }
                        else
                        {
                            delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds)));
                        }

                        Log.StartingRetryDelay(Logger, delayDuration);
                        await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false);

                        _nextRetryDelayMilliseconds = CalculateNextRetryDelay();

                        // Check if dispose was called on call.
                        CancellationTokenSource.Token.ThrowIfCancellationRequested();

                        // Clean up the failed call.
                        currentCall.Dispose();
                    }
                    else
                    {
                        // Handle the situation where the call failed with a non-deadline status, but retry
                        // didn't happen because of deadline exceeded.
                        IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded))
                            ? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus)
                            : currentCall;

                        // Can't retry.
                        // Signal public API exceptions that they should finish throwing and then exit the retry loop.
                        CommitCall(resolvedCall, result.GetValueOrDefault());
                        return;
                    }
                }
            }
            catch (Exception ex)
            {
                HandleUnexpectedError(ex);
            }
            finally
            {
                if (CommitedCallTask.IsCompletedSuccessfully())
                {
                    if (CommitedCallTask.Result is GrpcCall <TRequest, TResponse> call)
                    {
                        // Wait until the commited call is finished and then clean up retry call.
                        await call.CallTask.ConfigureAwait(false);

                        Cleanup();
                    }
                }

                Log.StoppingRetryWorker(Logger);
            }
        }
Esempio n. 19
0
        private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
        {
            GrpcCall <TRequest, TResponse>?call = null;

            try
            {
                lock (Lock)
                {
                    if (CommitedCallTask.IsCompletedSuccessfully())
                    {
                        // Call has already been commited. This could happen if written messages exceed
                        // buffer limits, which causes the call to immediately become commited and to clear buffers.
                        return;
                    }

                    OnStartingAttempt();

                    call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
                    _activeCalls.Add(call);

                    startCallFunc(call);

                    SetNewActiveCallUnsynchronized(call);

                    if (CommitedCallTask.IsCompletedSuccessfully())
                    {
                        // Call has already been commited. This could happen if written messages exceed
                        // buffer limits, which causes the call to immediately become commited and to clear buffers.
                        return;
                    }
                }

                Status?responseStatus;

                HttpResponseMessage?httpResponse = null;
                try
                {
                    httpResponse = await call.HttpResponseTask.ConfigureAwait(false);

                    responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
                }
                catch (RpcException ex)
                {
                    // A "drop" result from the load balancer should immediately stop the call,
                    // including ignoring the retry policy.
                    var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
                    if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
                    {
                        CommitCall(call, CommitReason.Drop);
                        return;
                    }

                    call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
                }
                catch (Exception ex)
                {
                    call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
                }

                CancellationTokenSource.Token.ThrowIfCancellationRequested();

                // Check to see the response returned from the server makes the call commited
                // Null status code indicates the headers were valid and a "Response-Headers" response
                // was received from the server.
                // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
                if (responseStatus == null)
                {
                    // Headers were returned. We're commited.
                    CommitCall(call, CommitReason.ResponseHeadersReceived);

                    // Wait until the call has finished and then check its status code
                    // to update retry throttling tokens.
                    var status = await call.CallTask.ConfigureAwait(false);

                    if (status.StatusCode == StatusCode.OK)
                    {
                        RetryAttemptCallSuccess();
                    }
                    return;
                }

                lock (Lock)
                {
                    var status = responseStatus.Value;
                    if (IsDeadlineExceeded())
                    {
                        // Deadline has been exceeded so immediately commit call.
                        CommitCall(call, CommitReason.DeadlineExceeded);
                    }
                    else if (!_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
                    {
                        CommitCall(call, CommitReason.FatalStatusCode);
                    }
                    else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
                    {
                        // This is the last active call and no more will be made.
                        CommitCall(call, CommitReason.ExceededAttemptCount);
                    }
                    else
                    {
                        // Call failed but it didn't exceed deadline, have a fatal status code
                        // and there are remaining attempts available. Is a chance it will be retried.
                        //
                        // Increment call failure out. Needs to happen before checking throttling.
                        RetryAttemptCallFailure();

                        if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
                        {
                            // This is the last active call and throttling is active.
                            CommitCall(call, CommitReason.Throttled);
                        }
                        else
                        {
                            var retryPushbackMS = GetRetryPushback(httpResponse);

                            // No need to interrupt if we started with no delay and all calls
                            // have already been made when hedging starting.
                            if (_delayInterruptTcs != null)
                            {
                                if (retryPushbackMS >= 0)
                                {
                                    _pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
                                }
                                _delayInterruptTcs.TrySetResult(null);
                            }

                            // Call isn't used and can be cancelled.
                            // Note that the call could have already been removed and disposed if the
                            // hedging call has been finalized or disposed.
                            if (_activeCalls.Remove(call))
                            {
                                call.Dispose();
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                HandleUnexpectedError(ex);
            }
            finally
            {
                if (CommitedCallTask.IsCompletedSuccessfully() && CommitedCallTask.Result == call)
                {
                    // Wait until the commited call is finished and then clean up hedging call.
                    await call.CallTask.ConfigureAwait(false);

                    Cleanup();
                }
            }
        }
Esempio n. 20
0
 public ValueTask DisposeAsync()
 {
     GrpcCall?.Dispose();
     return(default);
 public void Dispose() => GrpcCall?.Dispose();