private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
GrpcCall <TRequest, TResponse> call;
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
OnStartingAttempt();
call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
_activeCalls.Add(call);
startCallFunc(call);
SetNewActiveCallUnsynchronized(call);
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
call.CancellationToken.ThrowIfCancellationRequested();
CompatibilityExtensions.Assert(call._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled.");
httpResponse = await call._httpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (Exception ex)
{
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
if (CancellationTokenSource.IsCancellationRequested)
{
CommitCall(call, CommitReason.Canceled);
return;
}
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(call, CommitReason.ResponseHeadersReceived);
// Wait until the call has finished and then check its status code
// to update retry throttling tokens.
var status = await call.CallTask.ConfigureAwait(false);
if (status.StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
}
else
{
var status = responseStatus.Value;
var retryPushbackMS = GetRetryPushback(httpResponse);
if (retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
else if (_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
{
// Needs to happen before interrupt.
RetryAttemptCallFailure();
// No need to interrupt if we started with no delay and all calls
// have already been made when hedging starting.
if (_delayInterruptTcs != null)
{
lock (Lock)
{
if (retryPushbackMS >= 0)
{
_pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
}
_delayInterruptTcs.TrySetResult(null);
}
}
}
else
{
CommitCall(call, CommitReason.FatalStatusCode);
}
}
lock (Lock)
{
if (IsDeadlineExceeded())
{
// Deadline has been exceeded so immediately commit call.
CommitCall(call, CommitReason.DeadlineExceeded);
}
else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
{
// This is the last active call and no more will be made.
CommitCall(call, CommitReason.ExceededAttemptCount);
}
else if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
{
// This is the last active call and throttling is active.
CommitCall(call, CommitReason.Throttled);
}
else
{
// Call isn't used and can be cancelled.
// Note that the call could have already been removed and disposed if the
// hedging call has been finalized or disposed.
if (_activeCalls.Remove(call))
{
call.Dispose();
}
}
}
}
private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
Log.StartingRetryWorker(Logger);
try
{
// This is the main retry loop. It will:
// 1. Check the result of the active call was successful.
// 2. If it was unsuccessful then evaluate if the call can be retried.
// 3. If it can be retried then start a new active call and begin again.
while (true)
{
GrpcCall <TRequest, TResponse> currentCall;
lock (Lock)
{
// Start new call.
OnStartingAttempt();
currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
startCallFunc(currentCall);
SetNewActiveCallUnsynchronized(currentCall);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
httpResponse = await currentCall.HttpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (RpcException ex)
{
// A "drop" result from the load balancer should immediately stop the call,
// including ignoring the retry policy.
var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
{
CommitCall(currentCall, CommitReason.Drop);
return;
}
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
catch (Exception ex)
{
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited.
// 1. Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// 2. An OK response status at this point means a streaming call completed without
// sending any messages to the client.
//
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
responseStatus = await currentCall.CallTask.ConfigureAwait(false);
if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
// Commited so exit retry loop.
return;
}
else if (IsSuccessfulStreamingCall(responseStatus.GetValueOrDefault(), currentCall))
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
RetryAttemptCallSuccess();
// Commited so exit retry loop.
return;
}
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
var status = responseStatus.GetValueOrDefault();
var retryPushbackMS = GetRetryPushback(httpResponse);
// Failures only count towards retry throttling if they have a known, retriable status.
// This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling.
if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) ||
retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
var result = EvaluateRetry(status, retryPushbackMS);
Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null);
if (result == null)
{
TimeSpan delayDuration;
if (retryPushbackMS != null)
{
delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
_nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault();
}
else
{
delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds)));
}
Log.StartingRetryDelay(Logger, delayDuration);
await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false);
_nextRetryDelayMilliseconds = CalculateNextRetryDelay();
// Check if dispose was called on call.
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Clean up the failed call.
currentCall.Dispose();
}
else
{
// Handle the situation where the call failed with a non-deadline status, but retry
// didn't happen because of deadline exceeded.
IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded))
? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus)
: currentCall;
// Can't retry.
// Signal public API exceptions that they should finish throwing and then exit the retry loop.
CommitCall(resolvedCall, result.GetValueOrDefault());
return;
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
if (CommitedCallTask.Result is GrpcCall <TRequest, TResponse> call)
{
// Wait until the commited call is finished and then clean up retry call.
await call.CallTask.ConfigureAwait(false);
Cleanup();
}
}
Log.StoppingRetryWorker(Logger);
}
}
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
GrpcCall <TRequest, TResponse>?call = null;
try
{
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
OnStartingAttempt();
call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
_activeCalls.Add(call);
startCallFunc(call);
SetNewActiveCallUnsynchronized(call);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
httpResponse = await call.HttpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (RpcException ex)
{
// A "drop" result from the load balancer should immediately stop the call,
// including ignoring the retry policy.
var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
{
CommitCall(call, CommitReason.Drop);
return;
}
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
catch (Exception ex)
{
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(call, CommitReason.ResponseHeadersReceived);
// Wait until the call has finished and then check its status code
// to update retry throttling tokens.
var status = await call.CallTask.ConfigureAwait(false);
if (status.StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
return;
}
lock (Lock)
{
var status = responseStatus.Value;
if (IsDeadlineExceeded())
{
// Deadline has been exceeded so immediately commit call.
CommitCall(call, CommitReason.DeadlineExceeded);
}
else if (!_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
{
CommitCall(call, CommitReason.FatalStatusCode);
}
else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
{
// This is the last active call and no more will be made.
CommitCall(call, CommitReason.ExceededAttemptCount);
}
else
{
// Call failed but it didn't exceed deadline, have a fatal status code
// and there are remaining attempts available. Is a chance it will be retried.
//
// Increment call failure out. Needs to happen before checking throttling.
RetryAttemptCallFailure();
if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
{
// This is the last active call and throttling is active.
CommitCall(call, CommitReason.Throttled);
}
else
{
var retryPushbackMS = GetRetryPushback(httpResponse);
// No need to interrupt if we started with no delay and all calls
// have already been made when hedging starting.
if (_delayInterruptTcs != null)
{
if (retryPushbackMS >= 0)
{
_pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
}
_delayInterruptTcs.TrySetResult(null);
}
// Call isn't used and can be cancelled.
// Note that the call could have already been removed and disposed if the
// hedging call has been finalized or disposed.
if (_activeCalls.Remove(call))
{
call.Dispose();
}
}
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
if (CommitedCallTask.IsCompletedSuccessfully() && CommitedCallTask.Result == call)
{
// Wait until the commited call is finished and then clean up hedging call.
await call.CallTask.ConfigureAwait(false);
Cleanup();
}
}
}