protected override void OnCommitCall(IGrpcCall <TRequest, TResponse> call)
{
Debug.Assert(Monitor.IsEntered(Lock));
_activeCalls.Remove(call);
CleanUpUnsynchronized();
}
private Task <IGrpcCall <TRequest, TResponse>?> GetActiveCallAsync(IGrpcCall <TRequest, TResponse>?previousCall)
{
Debug.Assert(NewActiveCallTcs != null);
lock (Lock)
{
// Return currently active call if there is one, and its not the previous call.
if (_activeCall != null && previousCall != _activeCall)
{
return(Task.FromResult <IGrpcCall <TRequest, TResponse>?>(_activeCall));
}
// Wait to see whether new call will be made
return(GetActiveCallUnsynchronizedAsync(previousCall));
}
}
private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
Log.StartingRetryWorker(Logger);
try
{
// This is the main retry loop. It will:
// 1. Check the result of the active call was successful.
// 2. If it was unsuccessful then evaluate if the call can be retried.
// 3. If it can be retried then start a new active call and begin again.
while (true)
{
GrpcCall <TRequest, TResponse> currentCall;
lock (Lock)
{
// Start new call.
OnStartingAttempt();
currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
startCallFunc(currentCall);
SetNewActiveCallUnsynchronized(currentCall);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
httpResponse = await currentCall.HttpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (RpcException ex)
{
// A "drop" result from the load balancer should immediately stop the call,
// including ignoring the retry policy.
var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
{
CommitCall(currentCall, CommitReason.Drop);
return;
}
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
catch (Exception ex)
{
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited.
// 1. Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// 2. An OK response status at this point means a streaming call completed without
// sending any messages to the client.
//
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
responseStatus = await currentCall.CallTask.ConfigureAwait(false);
if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
// Commited so exit retry loop.
return;
}
else if (IsSuccessfulStreamingCall(responseStatus.GetValueOrDefault(), currentCall))
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
RetryAttemptCallSuccess();
// Commited so exit retry loop.
return;
}
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
var status = responseStatus.GetValueOrDefault();
var retryPushbackMS = GetRetryPushback(httpResponse);
// Failures only count towards retry throttling if they have a known, retriable status.
// This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling.
if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) ||
retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
var result = EvaluateRetry(status, retryPushbackMS);
Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null);
if (result == null)
{
TimeSpan delayDuration;
if (retryPushbackMS != null)
{
delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
_nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault();
}
else
{
delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds)));
}
Log.StartingRetryDelay(Logger, delayDuration);
await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false);
_nextRetryDelayMilliseconds = CalculateNextRetryDelay();
// Check if dispose was called on call.
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Clean up the failed call.
currentCall.Dispose();
}
else
{
// Handle the situation where the call failed with a non-deadline status, but retry
// didn't happen because of deadline exceeded.
IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded))
? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus)
: currentCall;
// Can't retry.
// Signal public API exceptions that they should finish throwing and then exit the retry loop.
CommitCall(resolvedCall, result.GetValueOrDefault());
return;
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
if (CommitedCallTask.Result is GrpcCall <TRequest, TResponse> call)
{
// Wait until the commited call is finished and then clean up retry call.
await call.CallTask.ConfigureAwait(false);
Cleanup();
}
}
Log.StoppingRetryWorker(Logger);
}
}
protected override void OnCommitCall(IGrpcCall <TRequest, TResponse> call)
{
_activeCall = null;
}
protected override void OnCommitCall(IGrpcCall <TRequest, TResponse> call)
{
_activeCalls.Remove(call);
CleanUpUnsynchronized();
}
