public async Task AsyncUnaryCall_FatalStatusCode_HedgeDelay_Failure() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, (callCount == 1) ? StatusCode.Unavailable : StatusCode.InvalidArgument)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(50)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode); Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode); Assert.AreEqual(2, callCount); }
public async Task Unary_AttemptsGreaterThanDefaultClientLimit_LimitedAttemptsMade(int hedgingDelay) { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 10, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelay))); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(5, callCount); AssertHasLog(LogLevel.Debug, "MaxAttemptsLimited", "The method has 10 attempts specified in the service config. The number of attempts has been limited by channel configuration to 5."); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount"); }
public async Task Unary_ExceedRetryAttempts_Failure() { Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { var metadata = new Metadata(); metadata.Add("grpc-retry-pushback-ms", "5"); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""), metadata))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig()); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); AssertHasLog(LogLevel.Debug, "RetryPushbackReceived", "Retry pushback of '5' received from the failed gRPC call."); AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 1, Retry: True"); AssertHasLog(LogLevel.Trace, "StartingRetryDelay", "Starting retry delay of 00:00:00.0050000."); AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 5, Retry: False"); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount"); }
public async Task Unary_ExceedAttempts_Failure(int?hedgingDelay) { Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Ignore errors SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var delay = (hedgingDelay == null) ? (TimeSpan?)null : TimeSpan.FromMilliseconds(hedgingDelay.Value); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: delay)); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount"); }
public async Task Unary_DeadlineExceedBeforeServerCall_Failure() { var callCount = 0; var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously); Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(tcs.Task); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow)); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(0, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded"); tcs.SetResult(new DataMessage()); }
public async Task ClientStreamWriter_WriteWhilePendingWrite_ErrorThrown() { // Arrange var httpClient = ClientTestHelpers.CreateTestClient(request => { var streamContent = new StreamContent(new SyncPointMemoryStream()); return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent))); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions()); // Assert var writeTask1 = call.RequestStream.WriteAsync(new HelloRequest { Name = "1" }); Assert.IsFalse(writeTask1.IsCompleted); var writeTask2 = call.RequestStream.WriteAsync(new HelloRequest { Name = "2" }); var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => writeTask2).DefaultTimeout(); Assert.AreEqual("Can't write the message because the previous write is in progress.", ex.Message); }
public async Task Unary_TriggerRetryThrottling_Failure() { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), retryThrottling: new RetryThrottlingPolicy { MaxTokens = 5, TokenRatio = 0.1 })); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(3, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Throttled"); }
public async Task AsyncUnaryCall_UnsupportedStatusCode_Failure() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callCount++; await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StringContent(""), StatusCode.InvalidArgument)); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode); Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode); }
public async Task AsyncUnaryCall_Success_RequestContentSent() { // Arrange HttpContent?content = null; var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callCount++; content = request.Content; var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message); }
public async Task AsyncUnaryCall_DisposeDuringBackoff_CanceledStatus() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callCount++; await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); var cts = new CancellationTokenSource(); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(cancellationToken: cts.Token), new HelloRequest { Name = "World" }); var delayTask = Task.Delay(100); var completedTask = await Task.WhenAny(call.ResponseAsync, delayTask); // Assert Assert.AreEqual(delayTask, completedTask); // Ensure that we're waiting for retry call.Dispose(); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); Assert.AreEqual("gRPC call disposed.", ex.Status.Detail); }
public async Task AsyncUnaryCall_FailureWithLongDelay_Dispose_CallImmediatelyDisposed() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callCount++; await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); }); // Very long delay var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(initialBackoff: TimeSpan.FromSeconds(30), maxBackoff: TimeSpan.FromSeconds(30)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); var resultTask = call.ResponseAsync; // Test will timeout if dispose doesn't kill the timer. call.Dispose(); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => resultTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); Assert.AreEqual("gRPC call disposed.", ex.Status.Detail); }
public async Task AsyncUnaryCall_SuccessAfterRetry_AccessResponseHeaders_SuccessfullyResponseHeadersReturned() { // Arrange HttpContent?content = null; var syncPoint = new SyncPoint(runContinuationsAsynchronously: true); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callCount++; content = request.Content !; if (callCount == 1) { await content.CopyToAsync(new MemoryStream()); await syncPoint.WaitForSyncPoint(); return(ResponseUtils.CreateHeadersOnlyResponse( HttpStatusCode.OK, StatusCode.Unavailable, customHeaders: new Dictionary <string, string> { ["call-count"] = callCount.ToString() })); } syncPoint.Continue(); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse( HttpStatusCode.OK, streamContent, customHeaders: new Dictionary <string, string> { ["call-count"] = callCount.ToString() })); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); var headersTask = call.ResponseHeadersAsync; // Wait until the first call has failed and the second is on the server await syncPoint.WaitToContinue().DefaultTimeout(); // Assert Assert.AreEqual(2, callCount); Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message); var headers = await headersTask.DefaultTimeout(); Assert.AreEqual("2", headers.GetValue("call-count")); }
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure() { // Arrange var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(request => { Interlocked.Increment(ref callCount); return(tcs.Task); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); }
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit, int hedgingDelayMilliseconds) { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Ignore errors SetExpectedErrorsFilter(writeContext => { if (writeContext.EventId.Name == "ErrorSendingMessage" || writeContext.EventId.Name == "ErrorExecutingServiceMethod") { return(true); } return(false); }); // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMilliseconds)), maxReceiveMessageSize: (int)GrpcChannel.DefaultMaxRetryBufferPerCallSize * 2); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage { Data = ByteString.CopyFrom(new byte[payloadSize]) }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); if (!exceedBufferLimit) { Assert.AreEqual(5, callCount); } else { Assert.AreEqual(1, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded"); // Cancelled calls could cause server errors. Delay so these error don't show up // in the next unit test. await Task.Delay(100); } Assert.AreEqual(0, channel.CurrentRetryBufferSize); }
public async Task ClientStreaming_WriteAsyncCancellationDuringRetry_Canceled(bool throwOperationCanceledOnCancellation) { async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { Logger.LogInformation("Server reading message 1."); Assert.IsTrue(await requestStream.MoveNext()); Logger.LogInformation("Server pausing."); await Task.Delay(TimeSpan.FromMilliseconds(500)); Logger.LogInformation("Server erroring."); throw new RpcException(new Status(StatusCode.Unavailable, string.Empty)); } SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromSeconds(20)), maxReceiveMessageSize: BigMessageSize * 2, maxRetryBufferPerCallSize: BigMessageSize * 2, throwOperationCanceledOnCancellation: throwOperationCanceledOnCancellation); var client = TestClientFactory.Create(channel, method); // Act var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); var call = client.ClientStreamingCall(); Logger.LogInformation("Client writing message 1."); await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }, cts.Token).DefaultTimeout(); Logger.LogInformation("Client writing message 2."); var writeTask = call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[BigMessageSize]) }, cts.Token); // Assert if (throwOperationCanceledOnCancellation) { var ex = await ExceptionAssert.ThrowsAsync <OperationCanceledException>(() => writeTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, call.GetStatus().StatusCode); } else { var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => writeTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); } Assert.IsTrue(cts.Token.IsCancellationRequested, "WriteAsync finished when CancellationToken wasn't triggered."); }
public async Task ClientStreaming_MultipleWritesAndRetries_Failure() { var nextFailure = 1; async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { List <byte> bytes = new List <byte>(); await foreach (var message in requestStream.ReadAllAsync()) { if (bytes.Count >= nextFailure) { nextFailure = nextFailure * 2; throw new RpcException(new Status(StatusCode.Unavailable, "")); } bytes.Add(message.Data[0]); } return(new DataMessage { Data = ByteString.CopyFrom(bytes.ToArray()) }); } SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(maxAttempts: 10), maxRetryAttempts: 10); var client = TestClientFactory.Create(channel, method); var sentData = new List <byte>(); // Act var call = client.ClientStreamingCall(); for (var i = 0; i < 20; i++) { sentData.Add((byte)i); await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)i }) }).DefaultTimeout(); await Task.Delay(1); } await call.RequestStream.CompleteAsync().DefaultTimeout(); var result = await call.ResponseAsync.DefaultTimeout(); // Assert Assert.IsTrue(result.Data.Span.SequenceEqual(sentData.ToArray())); }
public async Task AsyncServerStreamingCall_SuccessAfterRetry_RequestContentSent() { // Arrange var syncPoint = new SyncPoint(runContinuationsAsynchronously: true); MemoryStream?requestContent = null; var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); var s = await request.Content !.ReadAsStreamAsync(); var ms = new MemoryStream(); await s.CopyToAsync(ms); if (callCount == 1) { await syncPoint.WaitForSyncPoint(); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); } syncPoint.Continue(); requestContent = ms; var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(50)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None); // Wait until the first call has failed and the second is on the server await syncPoint.WaitToContinue().DefaultTimeout(); // Assert Assert.IsTrue(await moveNextTask); Assert.AreEqual("Hello world", call.ResponseStream.Current.Message); requestContent !.Seek(0, SeekOrigin.Begin); var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout(); Assert.AreEqual("World", requestMessage !.Name); }
public async Task AsyncUnaryCall_AuthInteceptorDispose_Error() { // Arrange var testSink = new TestSink(); var services = new ServiceCollection(); services.AddLogging(b => { b.AddProvider(new TestLoggerProvider(testSink)); }); services.AddNUnitLogger(); var provider = services.BuildServiceProvider(); var httpClient = ClientTestHelpers.CreateTestClient(async request => { var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var credentialsSyncPoint = new SyncPoint(runContinuationsAsynchronously: true); var credentials = CallCredentials.FromInterceptor(async(context, metadata) => { await credentialsSyncPoint.WaitToContinue(); metadata.Add("Authorization", $"Bearer TEST"); }); var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig, configure: options => options.Credentials = ChannelCredentials.Create(new SslCredentials(), credentials)); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); var responseTask = call.ResponseAsync; var responseHeadersTask = call.ResponseHeadersAsync; await credentialsSyncPoint.WaitForSyncPoint().DefaultTimeout(); call.Dispose(); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseHeadersTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited"); Assert.AreEqual("Call commited. Reason: Canceled", write.State.ToString()); }
public async Task Unary_RetryThrottlingBecomesActive_HasDelay_Failure() { var callCount = 0; var syncPoint = new SyncPoint(runContinuationsAsynchronously: true); async Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); await syncPoint.WaitToContinue(); return(request); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromMilliseconds(100), retryThrottling: new RetryThrottlingPolicy { MaxTokens = 5, TokenRatio = 0.1 })); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); await syncPoint.WaitForSyncPoint().DefaultTimeout(); // Manually trigger retry throttling Debug.Assert(channel.RetryThrottling != null); channel.RetryThrottling.CallFailure(); channel.RetryThrottling.CallFailure(); channel.RetryThrottling.CallFailure(); Debug.Assert(channel.RetryThrottling.IsRetryThrottlingActive()); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => HasLog(LogLevel.Debug, "AdditionalCallsBlockedByRetryThrottling", "Additional calls blocked by retry throttling."), "Check for expected log."); Assert.AreEqual(1, callCount); syncPoint.Continue(); await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ResponseHeadersReceived"); }
public async Task Dispose_ActiveCalls_CleansUpActiveCalls() { // Arrange var allCallsOnServerTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var waitUntilFinishedTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { // All calls are in-progress at once. Interlocked.Increment(ref callCount); if (callCount == 5) { allCallsOnServerTcs.SetResult(null); } await waitUntilFinishedTcs.Task; await request.Content !.CopyToAsync(new MemoryStream()); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMilliseconds(20)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); var hedgingCall = new HedgingCall <HelloRequest, HelloReply>(CreateHedgingPolicy(serviceConfig.MethodConfigs[0].HedgingPolicy), invoker.Channel, ClientTestHelpers.ServiceMethod, new CallOptions()); // Act hedgingCall.StartUnary(new HelloRequest { Name = "World" }); Assert.IsFalse(hedgingCall.CreateHedgingCallsTask !.IsCompleted); // Assert Assert.AreEqual(1, hedgingCall._activeCalls.Count); await allCallsOnServerTcs.Task.DefaultTimeout(); Assert.AreEqual(5, callCount); Assert.AreEqual(5, hedgingCall._activeCalls.Count); hedgingCall.Dispose(); Assert.AreEqual(0, hedgingCall._activeCalls.Count); await hedgingCall.CreateHedgingCallsTask !.DefaultTimeout(); waitUntilFinishedTcs.SetResult(null); }
public async Task AsyncServerStreamingCall_FailureAfterReadingResponseMessage_Failure() { // Arrange var streamContent = new SyncPointMemoryStream(); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(request => { callCount++; return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent)))); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest()); var responseStream = call.ResponseStream; // Assert Assert.IsNull(responseStream.Current); var moveNextTask1 = responseStream.MoveNext(CancellationToken.None); Assert.IsFalse(moveNextTask1.IsCompleted); await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply { Message = "Hello world 1" }).DefaultTimeout()).DefaultTimeout(); Assert.IsTrue(await moveNextTask1.DefaultTimeout()); Assert.IsNotNull(responseStream.Current); Assert.AreEqual("Hello world 1", responseStream.Current.Message); var moveNextTask2 = responseStream.MoveNext(CancellationToken.None); Assert.IsFalse(moveNextTask2.IsCompleted); await streamContent.AddExceptionAndWait(new Exception("Exception!")).DefaultTimeout(); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => moveNextTask2).DefaultTimeout(); Assert.AreEqual(StatusCode.Internal, ex.StatusCode); Assert.AreEqual(StatusCode.Internal, call.GetStatus().StatusCode); Assert.AreEqual("Error reading next message. Exception: Exception!", call.GetStatus().Detail); }
public async Task AsyncUnaryCall_ExceedAttempts_Failure() { // Arrange var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var requestMessages = new List <HelloRequest>(); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { // All calls are in-progress at once. Interlocked.Increment(ref callCount); if (callCount == 5) { tcs.TrySetResult(null); } await tcs.Task; var requestContent = await request.Content !.ReadAsStreamAsync(); var requestMessage = await ReadRequestMessage(requestContent); lock (requestMessages) { requestMessages.Add(requestMessage !); } return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(5, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(5, requestMessages.Count); foreach (var requestMessage in requestMessages) { Assert.AreEqual("World", requestMessage.Name); } }
public async Task Unary_MultipleLargeMessages_ExceedChannelMaxBufferSize() { // Arrange var sp1 = new SyncPoint(runContinuationsAsynchronously: true); var sp2 = new SyncPoint(runContinuationsAsynchronously: true); var sp3 = new SyncPoint(runContinuationsAsynchronously: true); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(), maxRetryBufferSize: 200, maxRetryBufferPerCallSize: 100); var request = new DataMessage { Data = ByteString.CopyFrom(new byte[90]) }; // Act var call1Task = MakeCall(Fixture, channel, request, sp1); await sp1.WaitForSyncPoint(); var call2Task = MakeCall(Fixture, channel, request, sp2); await sp2.WaitForSyncPoint(); // Will exceed channel buffer limit and won't retry var call3Task = MakeCall(Fixture, channel, request, sp3); await sp3.WaitForSyncPoint(); // Assert Assert.AreEqual(194, channel.CurrentRetryBufferSize); sp1.Continue(); sp2.Continue(); sp3.Continue(); var response = await call1Task.DefaultTimeout(); Assert.AreEqual(90, response.Data.Length); response = await call2Task.DefaultTimeout(); Assert.AreEqual(90, response.Data.Length); // Can't retry because buffer size exceeded. var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call3Task).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(0, channel.CurrentRetryBufferSize);
public async Task ServerStreaming_CancellatonTokenSpecified_TokenUnregisteredAndResourcesReleased() { Task FakeServerStreamCall(DataMessage request, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context) { return(Task.CompletedTask); } // Arrange var method = Fixture.DynamicGrpc.AddServerStreamingMethod <DataMessage, DataMessage>(FakeServerStreamCall); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var references = new List <WeakReference>(); // Checking that token register calls don't build up on CTS and create a memory leak. var cts = new CancellationTokenSource(); // Act // Send calls in a different method so there is no chance that a stack reference // to a gRPC call is still alive after calls are complete. await MakeCallsAsync(channel, method, references, cts.Token).DefaultTimeout(); // Assert // There is a race when cleaning up cancellation token registry. // Retry a few times to ensure GC is run after unregister. await TestHelpers.AssertIsTrueRetryAsync(() => { GC.Collect(); GC.WaitForPendingFinalizers(); for (var i = 0; i < references.Count; i++) { if (references[i].IsAlive) { return(false); } } // Resources for past calls were successfully GCed. return(true); }, "Assert that retry call resources are released."); }
public async Task Duplex_DeadlineExceedDuringDelay_Failure() { var callCount = 0; Task DuplexDeadlineExceeded(IAsyncStreamReader <DataMessage> requestStream, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context) { callCount++; return(Task.FromException(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata { new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()) }))); } // Arrange var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <DataMessage, DataMessage>(DuplexDeadlineExceeded); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var deadlineTimeout = 500; var call = client.DuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(deadlineTimeout))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseStream.MoveNext(CancellationToken.None)).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.RequestStream.WriteAsync(new DataMessage())).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(1, callCount); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); }
public async Task Unary_DeadlineExceedAfterServerCall_Failure(int exceptedServerCallCount) { var callCount = 0; var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously); Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; if (callCount < exceptedServerCallCount) { return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, "")))); } return(tcs.Task); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(200))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(exceptedServerCallCount, callCount); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded"); tcs.SetResult(new DataMessage()); }
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure() { // Arrange var testSink = new TestSink(); var services = new ServiceCollection(); services.AddLogging(b => { b.AddProvider(new TestLoggerProvider(testSink)); }); services.AddNUnitLogger(); var provider = services.BuildServiceProvider(); var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async(request, ct) => { // Ensure SendAsync call doesn't hang upon cancellation by gRPC client. using var registration = ct.Register(() => tcs.TrySetCanceled()); Interlocked.Increment(ref callCount); return(await tcs.Task); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited"); Assert.AreEqual("Call commited. Reason: DeadlineExceeded", write.State.ToString()); }
public async Task ClientStreamWriter_WriteWhileComplete_ErrorThrown() { // Arrange var streamContent = new SyncPointMemoryStream(); var httpClient = ClientTestHelpers.CreateTestClient(request => { return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent)))); }); var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions()); await call.RequestStream.CompleteAsync().DefaultTimeout(); var resultTask = call.ResponseAsync; // Assert var writeException1 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest { Name = "1" })).DefaultTimeout(); Assert.AreEqual("Request stream has already been completed.", writeException1.Message); await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply { Message = "Hello world 1" }).DefaultTimeout()).DefaultTimeout(); await streamContent.AddDataAndWait(new byte[0]); var result = await resultTask.DefaultTimeout(); Assert.AreEqual("Hello world 1", result.Message); var writeException2 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest { Name = "2" })).DefaultTimeout(); Assert.AreEqual("Request stream has already been completed.", writeException2.Message); }
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit) { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig()); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage { Data = ByteString.CopyFrom(new byte[payloadSize]) }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); if (!exceedBufferLimit) { Assert.AreEqual(5, callCount); } else { Assert.AreEqual(1, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded"); } Assert.AreEqual(0, channel.CurrentRetryBufferSize); }
public async Task Unary_DeadlineExceedDuringDelay_Failure() { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata { new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()) }))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(300))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(1, callCount); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded"); }