public override Task ClientStreamWriteAsync(TRequest message)
{
// The retry client stream writer prevents multiple threads from reaching here.
return(DoClientStreamActionAsync(async call =>
{
CompatibilityExtensions.Assert(call.ClientStreamWriter != null);
if (ClientStreamWriteOptions != null)
{
call.ClientStreamWriter.WriteOptions = ClientStreamWriteOptions;
}
await call.WriteClientStreamAsync(WriteNewMessage, message).ConfigureAwait(false);
lock (Lock)
{
BufferedCurrentMessage = false;
}
if (ClientStreamComplete)
{
await call.ClientStreamWriter.CompleteAsync().ConfigureAwait(false);
}
}));
}
private async Task HedgingDelayAsync(TimeSpan hedgingDelay)
{
CompatibilityExtensions.Assert(_hedgingDelayCts != null);
CompatibilityExtensions.Assert(_delayInterruptTcs != null);
while (true)
{
CompatibilityExtensions.Assert(_hedgingDelayCts != null);
var completedTask = await Task.WhenAny(Task.Delay(hedgingDelay, _hedgingDelayCts.Token), _delayInterruptTcs.Task).ConfigureAwait(false);
if (completedTask != _delayInterruptTcs.Task)
{
// Task.Delay won. Check CTS to see if it won because of cancellation.
_hedgingDelayCts.Token.ThrowIfCancellationRequested();
return;
}
else
{
// Cancel the Task.Delay that's no longer needed.
// https://github.com/davidfowl/AspNetCoreDiagnosticScenarios/blob/519ef7d231c01116f02bc04354816a735f2a36b6/AsyncGuidance.md#using-a-timeout
_hedgingDelayCts.Cancel();
}
lock (Lock)
{
// If we reaching this point then the delay was interrupted.
// Need to recreate the delay TCS/CTS for the next cycle.
_delayInterruptTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
_hedgingDelayCts = new CancellationTokenSource();
// Interrupt could come from a pushback, or a failing call with a non-fatal status.
if (_pushbackDelay != null)
{
// Use pushback value and delay again
hedgingDelay = _pushbackDelay.GetValueOrDefault();
_pushbackDelay = null;
}
else
{
// Immediately return for non-fatal status.
return;
}
}
}
}
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
GrpcCall <TRequest, TResponse> call;
lock (Lock)
{
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
OnStartingAttempt();
call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
_activeCalls.Add(call);
startCallFunc(call);
SetNewActiveCallUnsynchronized(call);
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
call.CancellationToken.ThrowIfCancellationRequested();
CompatibilityExtensions.Assert(call._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled.");
httpResponse = await call._httpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (Exception ex)
{
call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
if (CancellationTokenSource.IsCancellationRequested)
{
CommitCall(call, CommitReason.Canceled);
return;
}
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(call, CommitReason.ResponseHeadersReceived);
// Wait until the call has finished and then check its status code
// to update retry throttling tokens.
var status = await call.CallTask.ConfigureAwait(false);
if (status.StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
}
else
{
var status = responseStatus.Value;
var retryPushbackMS = GetRetryPushback(httpResponse);
if (retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
else if (_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode))
{
// Needs to happen before interrupt.
RetryAttemptCallFailure();
// No need to interrupt if we started with no delay and all calls
// have already been made when hedging starting.
if (_delayInterruptTcs != null)
{
lock (Lock)
{
if (retryPushbackMS >= 0)
{
_pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
}
_delayInterruptTcs.TrySetResult(null);
}
}
}
else
{
CommitCall(call, CommitReason.FatalStatusCode);
}
}
lock (Lock)
{
if (IsDeadlineExceeded())
{
// Deadline has been exceeded so immediately commit call.
CommitCall(call, CommitReason.DeadlineExceeded);
}
else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts)
{
// This is the last active call and no more will be made.
CommitCall(call, CommitReason.ExceededAttemptCount);
}
else if (_activeCalls.Count == 1 && IsRetryThrottlingActive())
{
// This is the last active call and throttling is active.
CommitCall(call, CommitReason.Throttled);
}
else
{
// Call isn't used and can be cancelled.
// Note that the call could have already been removed and disposed if the
// hedging call has been finalized or disposed.
if (_activeCalls.Remove(call))
{
call.Dispose();
}
}
}
}
public async Task AsyncUnaryCall_CompressMetadataSentWithRequest_RequestMessageCompressed(bool compressionDisabledOnOptions)
{
// Arrange
HttpRequestMessage?httpRequestMessage = null;
HelloRequest? helloRequest = null;
bool?isRequestNotCompressed = null;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
httpRequestMessage = request;
var requestData = await request.Content !.ReadAsByteArrayAsync().DefaultTimeout();
isRequestNotCompressed = requestData[0] == 0;
helloRequest = await StreamSerializationHelper.ReadMessageAsync(
new MemoryStream(requestData),
ClientTestHelpers.ServiceMethod.RequestMarshaller.ContextualDeserializer,
"gzip",
maximumMessageSize: null,
GrpcProtocolConstants.DefaultCompressionProviders,
singleMessage: true,
CancellationToken.None);
HelloReply reply = new HelloReply
{
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var compressionProviders = GrpcProtocolConstants.DefaultCompressionProviders.Values.ToList();
compressionProviders.Add(new TestCompressionProvider());
var invoker = HttpClientCallInvokerFactory.Create(httpClient, configure: o => o.CompressionProviders = compressionProviders);
var compressionMetadata = CreateClientCompressionMetadata("gzip");
var callOptions = new CallOptions(headers: compressionMetadata);
if (compressionDisabledOnOptions)
{
callOptions = callOptions.WithWriteOptions(new WriteOptions(WriteFlags.NoCompress));
}
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, callOptions, new HelloRequest
{
Name = "Hello"
});
// Assert
var response = await call.ResponseAsync;
Assert.IsNotNull(response);
Assert.AreEqual("Hello world", response.Message);
CompatibilityExtensions.Assert(httpRequestMessage != null);
Assert.AreEqual("identity,gzip,test", httpRequestMessage.Headers.GetValues(GrpcProtocolConstants.MessageAcceptEncodingHeader).Single());
Assert.AreEqual("gzip", httpRequestMessage.Headers.GetValues(GrpcProtocolConstants.MessageEncodingHeader).Single());
Assert.AreEqual(false, httpRequestMessage.Headers.Contains(GrpcProtocolConstants.CompressionRequestAlgorithmHeader));
CompatibilityExtensions.Assert(helloRequest != null);
Assert.AreEqual("Hello", helloRequest.Name);
Assert.AreEqual(compressionDisabledOnOptions, isRequestNotCompressed);
}
private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc)
{
Log.StartingRetryWorker(Logger);
try
{
// This is the main retry loop. It will:
// 1. Check the result of the active call was successful.
// 2. If it was unsuccessful then evaluate if the call can be retried.
// 3. If it can be retried then start a new active call and begin again.
while (true)
{
GrpcCall <TRequest, TResponse> currentCall;
lock (Lock)
{
// Start new call.
OnStartingAttempt();
currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount);
startCallFunc(currentCall);
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
SetNewActiveCallUnsynchronized(currentCall);
}
Status?responseStatus;
HttpResponseMessage?httpResponse = null;
try
{
currentCall.CancellationToken.ThrowIfCancellationRequested();
CompatibilityExtensions.Assert(currentCall._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled.");
httpResponse = await currentCall._httpResponseTask.ConfigureAwait(false);
responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _);
}
catch (Exception ex)
{
currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _);
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Check to see the response returned from the server makes the call commited
// Null status code indicates the headers were valid and a "Response-Headers" response
// was received from the server.
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (responseStatus == null)
{
// Headers were returned. We're commited.
CommitCall(currentCall, CommitReason.ResponseHeadersReceived);
responseStatus = await currentCall.CallTask.ConfigureAwait(false);
if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK)
{
RetryAttemptCallSuccess();
}
// Commited so exit retry loop.
return;
}
if (CommitedCallTask.IsCompletedSuccessfully())
{
// Call has already been commited. This could happen if written messages exceed
// buffer limits, which causes the call to immediately become commited and to clear buffers.
return;
}
Status status = responseStatus.Value;
var retryPushbackMS = GetRetryPushback(httpResponse);
// Failures only count towards retry throttling if they have a known, retriable status.
// This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling.
if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) ||
retryPushbackMS < 0)
{
RetryAttemptCallFailure();
}
var result = EvaluateRetry(status, retryPushbackMS);
Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null);
if (result == null)
{
TimeSpan delayDuration;
if (retryPushbackMS != null)
{
delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault());
_nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault();
}
else
{
delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds)));
}
Log.StartingRetryDelay(Logger, delayDuration);
await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false);
_nextRetryDelayMilliseconds = CalculateNextRetryDelay();
// Check if dispose was called on call.
CancellationTokenSource.Token.ThrowIfCancellationRequested();
// Clean up the failed call.
currentCall.Dispose();
}
else
{
// Handle the situation where the call failed with a non-deadline status, but retry
// didn't happen because of deadline exceeded.
IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded))
? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus)
: currentCall;
// Can't retry.
// Signal public API exceptions that they should finish throwing and then exit the retry loop.
CommitCall(resolvedCall, result.GetValueOrDefault());
return;
}
}
}
catch (Exception ex)
{
HandleUnexpectedError(ex);
}
finally
{
Log.StoppingRetryWorker(Logger);
}
}