public async Task MoveNext_TokenCanceledDuringCall_ThrowError() { // Arrange var cts = new CancellationTokenSource(); var httpClient = ClientTestHelpers.CreateTestClient(request => { var stream = new SyncPointMemoryStream(); var content = new StreamContent(stream); return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content))); }); var channel = GrpcChannel.ForAddress(httpClient.BaseAddress, new GrpcChannelOptions { HttpClient = httpClient }); var call = new GrpcCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, new CallOptions(), channel); call.StartServerStreaming(new HelloRequest()); // Act var moveNextTask1 = call.ClientStreamReader !.MoveNext(cts.Token); // Assert Assert.IsFalse(moveNextTask1.IsCompleted); cts.Cancel(); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => moveNextTask1).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); }
public WinHttpUnaryContent(TRequest request, Func <TRequest, Stream, ValueTask> startCallback, GrpcCall <TRequest, TResponse> call) { _request = request; _startCallback = startCallback; _call = call; Headers.ContentType = GrpcProtocolConstants.GrpcContentTypeHeaderValue; }
public async Task MoveNext_MultipleCallsWithoutAwait_ThrowError() { // Arrange var httpClient = ClientTestHelpers.CreateTestClient(request => { var stream = new SyncPointMemoryStream(); var content = new StreamContent(stream); return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content))); }); var channel = GrpcChannel.ForAddress(httpClient.BaseAddress, new GrpcChannelOptions { HttpClient = httpClient }); var call = new GrpcCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, new CallOptions(), channel); call.StartServerStreaming(new HelloRequest()); // Act var moveNextTask1 = call.ClientStreamReader !.MoveNext(CancellationToken.None); var moveNextTask2 = call.ClientStreamReader.MoveNext(CancellationToken.None); // Assert Assert.IsFalse(moveNextTask1.IsCompleted); var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => moveNextTask2).DefaultTimeout(); Assert.AreEqual("Can't read the next message because the previous read is still in progress.", ex.Message); }
static async ValueTask WriteBufferedMessages(GrpcCall <TRequest, TResponse> call, Stream requestStream, ReadOnlyMemory <byte>[] bufferedMessages) { foreach (var writtenMessage in bufferedMessages) { await call.WriteMessageAsync(requestStream, writtenMessage, call.CancellationToken).ConfigureAwait(false); } }
public void MoveNext_TokenCanceledDuringCall_ThrowError() { // Arrange var cts = new CancellationTokenSource(); var httpClient = TestHelpers.CreateTestClient(request => { var stream = new SyncPointMemoryStream(); var content = new StreamContent(stream); return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content))); }); var call = new GrpcCall <HelloRequest, HelloReply>(TestHelpers.ServiceMethod, new CallOptions(), SystemClock.Instance, NullLoggerFactory.Instance); call.StartServerStreaming(httpClient, new HelloRequest()); // Act var moveNextTask1 = call.ClientStreamReader !.MoveNext(cts.Token); // Assert Assert.IsFalse(moveNextTask1.IsCompleted); cts.Cancel(); var ex = Assert.ThrowsAsync <RpcException>(async() => await moveNextTask1.DefaultTimeout()); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); }
public void MoveNext_MultipleCallsWithoutAwait_ThrowError() { // Arrange var httpClient = TestHelpers.CreateTestClient(request => { var stream = new SyncPointMemoryStream(); var content = new StreamContent(stream); return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, content))); }); var call = new GrpcCall <HelloRequest, HelloReply>(TestHelpers.ServiceMethod, new CallOptions(), SystemClock.Instance, NullLoggerFactory.Instance); call.StartServerStreaming(httpClient, new HelloRequest()); // Act var moveNextTask1 = call.ClientStreamReader !.MoveNext(CancellationToken.None); var moveNextTask2 = call.ClientStreamReader.MoveNext(CancellationToken.None); // Assert Assert.IsFalse(moveNextTask1.IsCompleted); var ex = Assert.ThrowsAsync <InvalidOperationException>(async() => await moveNextTask2.DefaultTimeout()); Assert.AreEqual("Cannot read next message because the previous read is in progress.", ex.Message); }
private static bool IsSuccessfulStreamingCall(Status responseStatus, GrpcCall <TRequest, TResponse> call) { if (responseStatus.StatusCode != StatusCode.OK) { return(false); } return(call.Method.Type == MethodType.ServerStreaming || call.Method.Type == MethodType.DuplexStreaming); }
public async Task ReportsNewLeader(GrpcCall call) { EndPoint actual = default; var sut = new ReportLeaderInterceptor(ex => actual = ex); var result = await Assert.ThrowsAsync <NotLeaderException>(() => call(sut, Task.FromException <object>(new NotLeaderException("a.host", 2112)))); Assert.Equal(result.LeaderEndpoint, actual); }
private HttpContent CreatePushUnaryContent(TRequest request, GrpcCall <TRequest, TResponse> call) { return(!Channel.IsWinHttp ? new PushUnaryContent <TRequest, TResponse>(request, WriteAsync) : new WinHttpUnaryContent <TRequest, TResponse>(request, WriteAsync, call)); ValueTask WriteAsync(TRequest request, Stream stream) { return(WriteNewMessage(call, stream, call.Options, request)); } }
private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>( Method <TRequest, TResponse> method, CallOptions options) where TRequest : class where TResponse : class { if (_client.BaseAddress == null) { throw new InvalidOperationException("Unable to send the gRPC call because no server address has been configured. " + "Set HttpClient.BaseAddress on the HttpClient used to created to gRPC client."); } var call = new GrpcCall <TRequest, TResponse>(method, options, this); return(call); }
private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>( Method <TRequest, TResponse> method, CallOptions options, out Action disposeAction) where TRequest : class where TResponse : class { if (_client.BaseAddress == null) { throw new InvalidOperationException("Unable to send the gRPC call because no server address has been configured. " + "Set HttpClient.BaseAddress on the HttpClient used to created to gRPC client."); } CancellationTokenSource?linkedCts = null; // Use propagated deadline if it is small than the specified deadline if (Deadline < options.Deadline) { options = options.WithDeadline(Deadline); } if (CancellationToken.CanBeCanceled) { if (options.CancellationToken.CanBeCanceled) { // If both propagated and options cancellation token can be canceled // then set a new linked token of both linkedCts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, options.CancellationToken); options = options.WithCancellationToken(linkedCts.Token); } else { options = options.WithCancellationToken(CancellationToken); } } var call = new GrpcCall <TRequest, TResponse>(method, options, Clock, _loggerFactory); // Clean up linked cancellation token disposeAction = linkedCts != null ? () => { call.Dispose(); linkedCts !.Dispose(); } : (Action)call.Dispose; return(call); }
public static async Task <TResponse?> ReadMessageAsync <TResponse>( #endif this Stream responseStream, GrpcCall call, Func <DeserializationContext, TResponse> deserializer, string grpcEncoding, bool singleMessage, CancellationToken cancellationToken) where TResponse : class { byte[]? buffer = null; try { GrpcCallLog.ReadingMessage(call.Logger); cancellationToken.ThrowIfCancellationRequested(); // Buffer is used to read header, then message content. // This size was randomly chosen to hopefully be big enough for many small messages. // If the message is larger then the array will be replaced when the message size is known. buffer = ArrayPool <byte> .Shared.Rent(minimumLength : 4096); int read; var received = 0; while ((read = await responseStream.ReadAsync(buffer.AsMemory(received, GrpcProtocolConstants.HeaderSize - received), cancellationToken).ConfigureAwait(false)) > 0) { received += read; if (received == GrpcProtocolConstants.HeaderSize) { break; } } if (received < GrpcProtocolConstants.HeaderSize) { if (received == 0) { GrpcCallLog.NoMessageReturned(call.Logger); return(default);
private GrpcCall <TRequest, TResponse> CreateGrpcCall <TRequest, TResponse>( Method <TRequest, TResponse> method, CallOptions options, out Action disposeAction) where TRequest : class where TResponse : class { CancellationTokenSource?linkedCts = null; // Use propagated deadline if it is small than the specified deadline if (Deadline < options.Deadline) { options = options.WithDeadline(Deadline); } if (CancellationToken.CanBeCanceled) { if (options.CancellationToken.CanBeCanceled) { // If both propagated and options cancellation token can be canceled // then set a new linked token of both linkedCts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, options.CancellationToken); options = options.WithCancellationToken(linkedCts.Token); } else { options = options.WithCancellationToken(CancellationToken); } } var call = new GrpcCall <TRequest, TResponse>(method, options, Clock, _loggerFactory); // Clean up linked cancellation token disposeAction = linkedCts != null ? () => { call.Dispose(); linkedCts !.Dispose(); } : (Action)call.Dispose; return(call); }
private PushStreamContent <TRequest, TResponse> CreatePushStreamContent(GrpcCall <TRequest, TResponse> call, HttpContentClientStreamWriter <TRequest, TResponse> clientStreamWriter) { return(new PushStreamContent <TRequest, TResponse>(clientStreamWriter, async requestStream => { ValueTask writeTask; lock (Lock) { Log.SendingBufferedMessages(Logger, BufferedMessages.Count); if (BufferedMessages.Count == 0) { #if NETSTANDARD2_0 writeTask = Task.CompletedTask; #else writeTask = default; #endif } else if (BufferedMessages.Count == 1) { writeTask = call.WriteMessageAsync(requestStream, BufferedMessages[0], call.CancellationToken); } else { // Copy messages to a new collection in lock for thread-safety. var bufferedMessageCopy = BufferedMessages.ToArray(); writeTask = WriteBufferedMessages(call, requestStream, bufferedMessageCopy); } } await writeTask.ConfigureAwait(false); if (ClientStreamComplete) { await call.ClientStreamWriter !.CompleteAsync().ConfigureAwait(false); } }));
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc) { GrpcCall <TRequest, TResponse> call; lock (Lock) { if (CommitedCallTask.IsCompletedSuccessfully()) { // Call has already been commited. This could happen if written messages exceed // buffer limits, which causes the call to immediately become commited and to clear buffers. return; } OnStartingAttempt(); call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount); _activeCalls.Add(call); startCallFunc(call); SetNewActiveCallUnsynchronized(call); } Status?responseStatus; HttpResponseMessage?httpResponse = null; try { call.CancellationToken.ThrowIfCancellationRequested(); CompatibilityExtensions.Assert(call._httpResponseTask != null, "Request should have been made if call is not preemptively cancelled."); httpResponse = await call._httpResponseTask.ConfigureAwait(false); responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _); } catch (Exception ex) { call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _); } if (CancellationTokenSource.IsCancellationRequested) { CommitCall(call, CommitReason.Canceled); return; } // Check to see the response returned from the server makes the call commited // Null status code indicates the headers were valid and a "Response-Headers" response // was received from the server. // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid if (responseStatus == null) { // Headers were returned. We're commited. CommitCall(call, CommitReason.ResponseHeadersReceived); // Wait until the call has finished and then check its status code // to update retry throttling tokens. var status = await call.CallTask.ConfigureAwait(false); if (status.StatusCode == StatusCode.OK) { RetryAttemptCallSuccess(); } } else { var status = responseStatus.Value; var retryPushbackMS = GetRetryPushback(httpResponse); if (retryPushbackMS < 0) { RetryAttemptCallFailure(); } else if (_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode)) { // Needs to happen before interrupt. RetryAttemptCallFailure(); // No need to interrupt if we started with no delay and all calls // have already been made when hedging starting. if (_delayInterruptTcs != null) { lock (Lock) { if (retryPushbackMS >= 0) { _pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault()); } _delayInterruptTcs.TrySetResult(null); } } } else { CommitCall(call, CommitReason.FatalStatusCode); } } lock (Lock) { if (IsDeadlineExceeded()) { // Deadline has been exceeded so immediately commit call. CommitCall(call, CommitReason.DeadlineExceeded); } else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts) { // This is the last active call and no more will be made. CommitCall(call, CommitReason.ExceededAttemptCount); } else if (_activeCalls.Count == 1 && IsRetryThrottlingActive()) { // This is the last active call and throttling is active. CommitCall(call, CommitReason.Throttled); } else { // Call isn't used and can be cancelled. // Note that the call could have already been removed and disposed if the // hedging call has been finalized or disposed. if (_activeCalls.Remove(call)) { call.Dispose(); } } } }
public LengthUnaryContent(TRequest content, GrpcCall <TRequest, TResponse> call, MediaTypeHeaderValue mediaType) { _content = content; _call = call; Headers.ContentType = mediaType; }
protected override void OnCommitCall(IGrpcCall <TRequest, TResponse> call) { _activeCall = null; }
private async Task StartRetry(Action <GrpcCall <TRequest, TResponse> > startCallFunc) { Log.StartingRetryWorker(Logger); try { // This is the main retry loop. It will: // 1. Check the result of the active call was successful. // 2. If it was unsuccessful then evaluate if the call can be retried. // 3. If it can be retried then start a new active call and begin again. while (true) { GrpcCall <TRequest, TResponse> currentCall; lock (Lock) { // Start new call. OnStartingAttempt(); currentCall = _activeCall = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount); startCallFunc(currentCall); SetNewActiveCallUnsynchronized(currentCall); if (CommitedCallTask.IsCompletedSuccessfully()) { // Call has already been commited. This could happen if written messages exceed // buffer limits, which causes the call to immediately become commited and to clear buffers. return; } } Status?responseStatus; HttpResponseMessage?httpResponse = null; try { httpResponse = await currentCall.HttpResponseTask.ConfigureAwait(false); responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _); } catch (RpcException ex) { // A "drop" result from the load balancer should immediately stop the call, // including ignoring the retry policy. var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer); if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop) { CommitCall(currentCall, CommitReason.Drop); return; } currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _); } catch (Exception ex) { currentCall.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _); } CancellationTokenSource.Token.ThrowIfCancellationRequested(); // Check to see the response returned from the server makes the call commited. // 1. Null status code indicates the headers were valid and a "Response-Headers" response // was received from the server. // 2. An OK response status at this point means a streaming call completed without // sending any messages to the client. // // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid if (responseStatus == null) { // Headers were returned. We're commited. CommitCall(currentCall, CommitReason.ResponseHeadersReceived); responseStatus = await currentCall.CallTask.ConfigureAwait(false); if (responseStatus.GetValueOrDefault().StatusCode == StatusCode.OK) { RetryAttemptCallSuccess(); } // Commited so exit retry loop. return; } else if (IsSuccessfulStreamingCall(responseStatus.GetValueOrDefault(), currentCall)) { // Headers were returned. We're commited. CommitCall(currentCall, CommitReason.ResponseHeadersReceived); RetryAttemptCallSuccess(); // Commited so exit retry loop. return; } if (CommitedCallTask.IsCompletedSuccessfully()) { // Call has already been commited. This could happen if written messages exceed // buffer limits, which causes the call to immediately become commited and to clear buffers. return; } var status = responseStatus.GetValueOrDefault(); var retryPushbackMS = GetRetryPushback(httpResponse); // Failures only count towards retry throttling if they have a known, retriable status. // This stops non-transient statuses, e.g. INVALID_ARGUMENT, from triggering throttling. if (_retryPolicy.RetryableStatusCodes.Contains(status.StatusCode) || retryPushbackMS < 0) { RetryAttemptCallFailure(); } var result = EvaluateRetry(status, retryPushbackMS); Log.RetryEvaluated(Logger, status.StatusCode, AttemptCount, result == null); if (result == null) { TimeSpan delayDuration; if (retryPushbackMS != null) { delayDuration = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault()); _nextRetryDelayMilliseconds = retryPushbackMS.GetValueOrDefault(); } else { delayDuration = TimeSpan.FromMilliseconds(Channel.GetRandomNumber(0, Convert.ToInt32(_nextRetryDelayMilliseconds))); } Log.StartingRetryDelay(Logger, delayDuration); await Task.Delay(delayDuration, CancellationTokenSource.Token).ConfigureAwait(false); _nextRetryDelayMilliseconds = CalculateNextRetryDelay(); // Check if dispose was called on call. CancellationTokenSource.Token.ThrowIfCancellationRequested(); // Clean up the failed call. currentCall.Dispose(); } else { // Handle the situation where the call failed with a non-deadline status, but retry // didn't happen because of deadline exceeded. IGrpcCall <TRequest, TResponse> resolvedCall = (IsDeadlineExceeded() && !(currentCall.CallTask.IsCompletedSuccessfully() && currentCall.CallTask.Result.StatusCode == StatusCode.DeadlineExceeded)) ? CreateStatusCall(GrpcProtocolConstants.DeadlineExceededStatus) : currentCall; // Can't retry. // Signal public API exceptions that they should finish throwing and then exit the retry loop. CommitCall(resolvedCall, result.GetValueOrDefault()); return; } } } catch (Exception ex) { HandleUnexpectedError(ex); } finally { if (CommitedCallTask.IsCompletedSuccessfully()) { if (CommitedCallTask.Result is GrpcCall <TRequest, TResponse> call) { // Wait until the commited call is finished and then clean up retry call. await call.CallTask.ConfigureAwait(false); Cleanup(); } } Log.StoppingRetryWorker(Logger); } }
private async Task StartCall(Action <GrpcCall <TRequest, TResponse> > startCallFunc) { GrpcCall <TRequest, TResponse>?call = null; try { lock (Lock) { if (CommitedCallTask.IsCompletedSuccessfully()) { // Call has already been commited. This could happen if written messages exceed // buffer limits, which causes the call to immediately become commited and to clear buffers. return; } OnStartingAttempt(); call = HttpClientCallInvoker.CreateGrpcCall <TRequest, TResponse>(Channel, Method, Options, AttemptCount); _activeCalls.Add(call); startCallFunc(call); SetNewActiveCallUnsynchronized(call); if (CommitedCallTask.IsCompletedSuccessfully()) { // Call has already been commited. This could happen if written messages exceed // buffer limits, which causes the call to immediately become commited and to clear buffers. return; } } Status?responseStatus; HttpResponseMessage?httpResponse = null; try { httpResponse = await call.HttpResponseTask.ConfigureAwait(false); responseStatus = GrpcCall.ValidateHeaders(httpResponse, out _); } catch (RpcException ex) { // A "drop" result from the load balancer should immediately stop the call, // including ignoring the retry policy. var dropValue = ex.Trailers.GetValue(GrpcProtocolConstants.DropRequestTrailer); if (dropValue != null && bool.TryParse(dropValue, out var isDrop) && isDrop) { CommitCall(call, CommitReason.Drop); return; } call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _); } catch (Exception ex) { call.ResolveException(GrpcCall <TRequest, TResponse> .ErrorStartingCallMessage, ex, out responseStatus, out _); } CancellationTokenSource.Token.ThrowIfCancellationRequested(); // Check to see the response returned from the server makes the call commited // Null status code indicates the headers were valid and a "Response-Headers" response // was received from the server. // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid if (responseStatus == null) { // Headers were returned. We're commited. CommitCall(call, CommitReason.ResponseHeadersReceived); // Wait until the call has finished and then check its status code // to update retry throttling tokens. var status = await call.CallTask.ConfigureAwait(false); if (status.StatusCode == StatusCode.OK) { RetryAttemptCallSuccess(); } return; } lock (Lock) { var status = responseStatus.Value; if (IsDeadlineExceeded()) { // Deadline has been exceeded so immediately commit call. CommitCall(call, CommitReason.DeadlineExceeded); } else if (!_hedgingPolicy.NonFatalStatusCodes.Contains(status.StatusCode)) { CommitCall(call, CommitReason.FatalStatusCode); } else if (_activeCalls.Count == 1 && AttemptCount >= MaxRetryAttempts) { // This is the last active call and no more will be made. CommitCall(call, CommitReason.ExceededAttemptCount); } else { // Call failed but it didn't exceed deadline, have a fatal status code // and there are remaining attempts available. Is a chance it will be retried. // // Increment call failure out. Needs to happen before checking throttling. RetryAttemptCallFailure(); if (_activeCalls.Count == 1 && IsRetryThrottlingActive()) { // This is the last active call and throttling is active. CommitCall(call, CommitReason.Throttled); } else { var retryPushbackMS = GetRetryPushback(httpResponse); // No need to interrupt if we started with no delay and all calls // have already been made when hedging starting. if (_delayInterruptTcs != null) { if (retryPushbackMS >= 0) { _pushbackDelay = TimeSpan.FromMilliseconds(retryPushbackMS.GetValueOrDefault()); } _delayInterruptTcs.TrySetResult(null); } // Call isn't used and can be cancelled. // Note that the call could have already been removed and disposed if the // hedging call has been finalized or disposed. if (_activeCalls.Remove(call)) { call.Dispose(); } } } } } catch (Exception ex) { HandleUnexpectedError(ex); } finally { if (CommitedCallTask.IsCompletedSuccessfully() && CommitedCallTask.Result == call) { // Wait until the commited call is finished and then clean up hedging call. await call.CallTask.ConfigureAwait(false); Cleanup(); } } }
public ValueTask DisposeAsync() { GrpcCall?.Dispose(); return(default);
public void Dispose() => GrpcCall?.Dispose();