protected override void StartCore(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
var hedgingDelay = _hedgingPolicy.HedgingDelay;
if (hedgingDelay == TimeSpan.Zero)
{
// If there is no delay then start all call immediately
while (AttemptCount < MaxRetryAttempts)
{
_ = StartCall(startCallFunc);
// Don't send additional calls if retry throttling is active.
if (IsRetryThrottlingActive())
{
Log.AdditionalCallsBlockedByRetryThrottling(Logger);
break;
}
lock (Lock)
{
// Don't send additional calls if call has been commited.
if (CommitedCallTask.IsCompletedSuccessfully())
{
break;
}
}
}
}
else
{
CreateHedgingCallsTask = CreateHedgingCalls(startCallFunc);
}
}
public Metadata GetTrailers()
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
return(CommitedCallTask.Result.GetTrailers());
}
throw new InvalidOperationException("Can't get the call trailers because the call has not completed successfully.");
}
public Status GetStatus()
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
return(CommitedCallTask.Result.GetStatus());
}
throw new InvalidOperationException("Unable to get the status because the call is not complete.");
}
private async Task CreateHedgingCalls(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
Log.StartingRetryWorker(Logger);
try
{
var hedgingDelay = _hedgingPolicy.HedgingDelay;
while (AttemptCount < MaxRetryAttempts)
{
_ = StartCall(startCallFunc);
await HedgingDelayAsync(hedgingDelay).ConfigureAwait(false);
if (IsDeadlineExceeded())
{
CommitCall(new StatusGrpcCall <TRequest, TResponse>(new Status(StatusCode.DeadlineExceeded, string.Empty)), CommitReason.DeadlineExceeded);
break;
}
else
{
lock (Lock)
{
if (IsRetryThrottlingActive())
{
if (_activeCalls.Count == 0)
{
CommitCall(CreateStatusCall(GrpcProtocolConstants.ThrottledStatus), CommitReason.Throttled);
}
else
{
Log.AdditionalCallsBlockedByRetryThrottling(Logger);
}
break;
}
// Don't send additional calls if call has been commited.
if (CommitedCallTask.IsCompletedSuccessfully())
{
break;
}
}
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
Log.StoppingRetryWorker(Logger);
}
}
public async Task <Metadata> GetResponseHeadersAsync()
{
var call = await CommitedCallTask.ConfigureAwait(false);
return(await call.GetResponseHeadersAsync().ConfigureAwait(false));
}
public async Task <TResponse> GetResponseAsync()
{
var call = await CommitedCallTask.ConfigureAwait(false);
return(await call.GetResponseAsync().ConfigureAwait(false));
}
private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
Log.StartingRetryWorker(Logger);
try
{
// This is the main retry loop. It will:
// 1. Check the result of the active call was successful.
// 2. If it was unsuccessful then evaluate if the call can be retried.
// 3. If it can be retried then start a new active call and begin again.
while (true)
{
GrpcCall <TRequest, TResponse> currentCall;
lock (Lock)
{
// Start new call.
OnStartingAttempt();
currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
startCallFunc(currentCall);
SetNewActiveCallUnsynchronized(currentCall);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
httpResponse = await currentCall.HttpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (RpcException ex)
{
// A "drop" result from the load balancer should immediately stop the call,
// including ignoring the retry policy.
var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
{
CommitCall(currentCall, CommitReason.Drop);
return;
}
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
catch (Exception ex)
{
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited.
// 1. Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// 2. An OK response status at this point means a streaming call completed without
// sending any messages to the client.
//
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
responseStatus = await currentCall.CallTask.ConfigureAwait(false);
if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
// Commited so exit retry loop.
return;
}
else if (IsSuccessfulStreamingCall(responseStatus.GetValueOrDefault(), currentCall))
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
RetryAttemptCallSuccess();
// Commited so exit retry loop.
return;
}
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
var status = responseStatus.GetValueOrDefault();
var retryPushbackMS = GetRetryPushback(httpResponse);
// Failures only count towards retry throttling if they have a known, retriable status.
// This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling.
if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) ||
retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
var result = EvaluateRetry(status, retryPushbackMS);
Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null);
if (result == null)
{
TimeSpan delayDuration;
if (retryPushbackMS != null)
{
delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
_nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault();
}
else
{
delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds)));
}
Log.StartingRetryDelay(Logger, delayDuration);
await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false);
_nextRetryDelayMilliseconds = CalculateNextRetryDelay();
// Check if dispose was called on call.
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Clean up the failed call.
currentCall.Dispose();
}
else
{
// Handle the situation where the call failed with a non-deadline status, but retry
// didn't happen because of deadline exceeded.
IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded))
? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus)
: currentCall;
// Can't retry.
// Signal public API exceptions that they should finish throwing and then exit the retry loop.
CommitCall(resolvedCall, result.GetValueOrDefault());
return;
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
if (CommitedCallTask.Result is GrpcCall <TRequest, TResponse> call)
{
// Wait until the commited call is finished and then clean up retry call.
await call.CallTask.ConfigureAwait(false);
Cleanup();
}
}
Log.StoppingRetryWorker(Logger);
}
}
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
GrpcCall <TRequest, TResponse> call;
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
OnStartingAttempt();
call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
_activeCalls.Add(call);
startCallFunc(call);
SetNewActiveCallUnsynchronized(call);
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
call.CancellationToken.ThrowIfCancellationRequested();
CompatibilityExtensions.Assert(call._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled.");
httpResponse = await call._httpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (Exception ex)
{
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
if (CancellationTokenSource.IsCancellationRequested)
{
CommitCall(call, CommitReason.Canceled);
return;
}
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(call, CommitReason.ResponseHeadersReceived);
// Wait until the call has finished and then check its status code
// to update retry throttling tokens.
var status = await call.CallTask.ConfigureAwait(false);
if (status.StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
}
else
{
var status = responseStatus.Value;
var retryPushbackMS = GetRetryPushback(httpResponse);
if (retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
else if (_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
{
// Needs to happen before interrupt.
RetryAttemptCallFailure();
// No need to interrupt if we started with no delay and all calls
// have already been made when hedging starting.
if (_delayInterruptTcs != null)
{
lock (Lock)
{
if (retryPushbackMS >= 0)
{
_pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
}
_delayInterruptTcs.TrySetResult(null);
}
}
}
else
{
CommitCall(call, CommitReason.FatalStatusCode);
}
}
lock (Lock)
{
if (IsDeadlineExceeded())
{
// Deadline has been exceeded so immediately commit call.
CommitCall(call, CommitReason.DeadlineExceeded);
}
else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
{
// This is the last active call and no more will be made.
CommitCall(call, CommitReason.ExceededAttemptCount);
}
else if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
{
// This is the last active call and throttling is active.
CommitCall(call, CommitReason.Throttled);
}
else
{
// Call isn't used and can be cancelled.
// Note that the call could have already been removed and disposed if the
// hedging call has been finalized or disposed.
if (_activeCalls.Remove(call))
{
call.Dispose();
}
}
}
}
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
GrpcCall <TRequest, TResponse>?call = null;
try
{
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
OnStartingAttempt();
call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
_activeCalls.Add(call);
startCallFunc(call);
SetNewActiveCallUnsynchronized(call);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
httpResponse = await call.HttpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (RpcException ex)
{
// A "drop" result from the load balancer should immediately stop the call,
// including ignoring the retry policy.
var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer);
if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop)
{
CommitCall(call, CommitReason.Drop);
return;
}
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
catch (Exception ex)
{
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(call, CommitReason.ResponseHeadersReceived);
// Wait until the call has finished and then check its status code
// to update retry throttling tokens.
var status = await call.CallTask.ConfigureAwait(false);
if (status.StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
return;
}
lock (Lock)
{
var status = responseStatus.Value;
if (IsDeadlineExceeded())
{
// Deadline has been exceeded so immediately commit call.
CommitCall(call, CommitReason.DeadlineExceeded);
}
else if (!_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
{
CommitCall(call, CommitReason.FatalStatusCode);
}
else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
{
// This is the last active call and no more will be made.
CommitCall(call, CommitReason.ExceededAttemptCount);
}
else
{
// Call failed but it didn't exceed deadline, have a fatal status code
// and there are remaining attempts available. Is a chance it will be retried.
//
// Increment call failure out. Needs to happen before checking throttling.
RetryAttemptCallFailure();
if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
{
// This is the last active call and throttling is active.
CommitCall(call, CommitReason.Throttled);
}
else
{
var retryPushbackMS = GetRetryPushback(httpResponse);
// No need to interrupt if we started with no delay and all calls
// have already been made when hedging starting.
if (_delayInterruptTcs != null)
{
if (retryPushbackMS >= 0)
{
_pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
}
_delayInterruptTcs.TrySetResult(null);
}
// Call isn't used and can be cancelled.
// Note that the call could have already been removed and disposed if the
// hedging call has been finalized or disposed.
if (_activeCalls.Remove(call))
{
call.Dispose();
}
}
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
if (CommitedCallTask.IsCompletedSuccessfully() && CommitedCallTask.Result == call)
{
// Wait until the commited call is finished and then clean up hedging call.
await call.CallTask.ConfigureAwait(false);
Cleanup();
}
}
}
public override async Task ClientStreamWriteAsync(TRequest message, CancellationToken cancellationToken)
{
// The retry client stream writer prevents multiple threads from reaching here.
// Hedging allows:
// 1. Multiple calls to happen simultaniously
// 2. Starting a new call once existing calls have failed.
//
// We don't want to exit this method until either the message has been sent to the
// server at least once successfully, or the call fails.
// If there is an error sending the message then wait for another call to successfully
// send the buffered data.
//
// This is done by awaiting a TCS until either:
// 1. The message count is sent.
// 2. The call is commited with an error, throwing an exception.
_writeClientMessageCount++;
_writeClientMessageTcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
// Register after TCS is created so immediate failure is propagated to TCS.
using var registration = (cancellationToken.CanBeCanceled && cancellationToken != Options.CancellationToken)
? RegisterRetryCancellationToken(cancellationToken)
: default;
try
{
await DoClientStreamActionAsync(async calls =>
{
// Note: There may be less write tasks than calls passed in.
// For example, a large message could cause the call to be commited and all active calls are removed.
var writeTasks = new List <Task>(calls.Count);
List <CancellationTokenRegistration>?registrations = null;
for (var i = 0; i < calls.Count; i++)
{
var c = calls[i];
if (c.TryRegisterCancellation(cancellationToken, out var registration))
{
registrations ??= new List <CancellationTokenRegistration>(calls.Count);
registrations.Add(registration.GetValueOrDefault());
}
var writeTask = c.WriteClientStreamAsync(WriteNewMessage, message);
writeTasks.Add(writeTask);
}
try
{
await Task.WhenAll(writeTasks).ConfigureAwait(false);
}
finally
{
if (registrations != null)
{
foreach (var registration in registrations)
{
registration.Dispose();
}
}
}
}).ConfigureAwait(false);
}
catch
{
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
throw;
}
}
// Flag indicates whether buffered message was successfully written.
var success = await _writeClientMessageTcs.Task.ConfigureAwait(false);
if (success)
{
return;
}
else
{
var commitedCall = CommitedCallTask.Result;
throw commitedCall.CreateFailureStatusException(commitedCall.GetStatus());
}
}
finally
{
_writeClientMessageTcs = null;
}
lock (Lock)
{
BufferedCurrentMessage = false;
}
}
