public async Task Unary_ExceedAttempts_Failure(int?hedgingDelay) { Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Ignore errors SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var delay = (hedgingDelay == null) ? (TimeSpan?)null : TimeSpan.FromMilliseconds(hedgingDelay.Value); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: delay)); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount"); }
public async Task Unary_DeadlineExceedBeforeServerCall_Failure() { var callCount = 0; var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously); Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(tcs.Task); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow)); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(0, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded"); tcs.SetResult(new DataMessage()); }
public async Task Unary_TriggerRetryThrottling_Failure() { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), retryThrottling: new RetryThrottlingPolicy { MaxTokens = 5, TokenRatio = 0.1 })); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(3, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Throttled"); }
public async Task Unary_AttemptsGreaterThanDefaultClientLimit_LimitedAttemptsMade(int hedgingDelay) { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 10, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelay))); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(5, callCount); AssertHasLog(LogLevel.Debug, "MaxAttemptsLimited", "The method has 10 attempts specified in the service config. The number of attempts has been limited by channel configuration to 5."); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount"); }
public async Task AsyncUnaryCall_FatalStatusCode_HedgeDelay_Failure() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, (callCount == 1) ? StatusCode.Unavailable : StatusCode.InvalidArgument)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(50)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode); Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode); Assert.AreEqual(2, callCount); }
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure() { // Arrange var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(request => { Interlocked.Increment(ref callCount); return(tcs.Task); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); }
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit, int hedgingDelayMilliseconds) { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, "")))); } // Ignore errors SetExpectedErrorsFilter(writeContext => { if (writeContext.EventId.Name == "ErrorSendingMessage" || writeContext.EventId.Name == "ErrorExecutingServiceMethod") { return(true); } return(false); }); // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMilliseconds)), maxReceiveMessageSize: (int)GrpcChannel.DefaultMaxRetryBufferPerCallSize * 2); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage { Data = ByteString.CopyFrom(new byte[payloadSize]) }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); if (!exceedBufferLimit) { Assert.AreEqual(5, callCount); } else { Assert.AreEqual(1, callCount); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded"); // Cancelled calls could cause server errors. Delay so these error don't show up // in the next unit test. await Task.Delay(100); } Assert.AreEqual(0, channel.CurrentRetryBufferSize); }
public async Task ClientStreaming_WriteAsyncCancellationDuringRetry_Canceled(bool throwOperationCanceledOnCancellation) { async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { Logger.LogInformation("Server reading message 1."); Assert.IsTrue(await requestStream.MoveNext()); Logger.LogInformation("Server pausing."); await Task.Delay(TimeSpan.FromMilliseconds(500)); Logger.LogInformation("Server erroring."); throw new RpcException(new Status(StatusCode.Unavailable, string.Empty)); } SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromSeconds(20)), maxReceiveMessageSize: BigMessageSize * 2, maxRetryBufferPerCallSize: BigMessageSize * 2, throwOperationCanceledOnCancellation: throwOperationCanceledOnCancellation); var client = TestClientFactory.Create(channel, method); // Act var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); var call = client.ClientStreamingCall(); Logger.LogInformation("Client writing message 1."); await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }, cts.Token).DefaultTimeout(); Logger.LogInformation("Client writing message 2."); var writeTask = call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[BigMessageSize]) }, cts.Token); // Assert if (throwOperationCanceledOnCancellation) { var ex = await ExceptionAssert.ThrowsAsync <OperationCanceledException>(() => writeTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, call.GetStatus().StatusCode); } else { var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => writeTask).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode); } Assert.IsTrue(cts.Token.IsCancellationRequested, "WriteAsync finished when CancellationToken wasn't triggered."); }
public async Task AsyncServerStreamingCall_SuccessAfterRetry_RequestContentSent() { // Arrange var syncPoint = new SyncPoint(runContinuationsAsynchronously: true); MemoryStream?requestContent = null; var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); var s = await request.Content !.ReadAsStreamAsync(); var ms = new MemoryStream(); await s.CopyToAsync(ms); if (callCount == 1) { await syncPoint.WaitForSyncPoint(); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); } syncPoint.Continue(); requestContent = ms; var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(50)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None); // Wait until the first call has failed and the second is on the server await syncPoint.WaitToContinue().DefaultTimeout(); // Assert Assert.IsTrue(await moveNextTask); Assert.AreEqual("Hello world", call.ResponseStream.Current.Message); requestContent !.Seek(0, SeekOrigin.Begin); var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout(); Assert.AreEqual("World", requestMessage !.Name); }
public async Task Unary_RetryThrottlingBecomesActive_HasDelay_Failure() { var callCount = 0; var syncPoint = new SyncPoint(runContinuationsAsynchronously: true); async Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { Interlocked.Increment(ref callCount); await syncPoint.WaitToContinue(); return(request); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromMilliseconds(100), retryThrottling: new RetryThrottlingPolicy { MaxTokens = 5, TokenRatio = 0.1 })); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage()); await syncPoint.WaitForSyncPoint().DefaultTimeout(); // Manually trigger retry throttling Debug.Assert(channel.RetryThrottling != null); channel.RetryThrottling.CallFailure(); channel.RetryThrottling.CallFailure(); channel.RetryThrottling.CallFailure(); Debug.Assert(channel.RetryThrottling.IsRetryThrottlingActive()); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => HasLog(LogLevel.Debug, "AdditionalCallsBlockedByRetryThrottling", "Additional calls blocked by retry throttling."), "Check for expected log."); Assert.AreEqual(1, callCount); syncPoint.Continue(); await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ResponseHeadersReceived"); }
public async Task Dispose_ActiveCalls_CleansUpActiveCalls() { // Arrange var allCallsOnServerTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var waitUntilFinishedTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { // All calls are in-progress at once. Interlocked.Increment(ref callCount); if (callCount == 5) { allCallsOnServerTcs.SetResult(null); } await waitUntilFinishedTcs.Task; await request.Content !.CopyToAsync(new MemoryStream()); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMilliseconds(20)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); var hedgingCall = new HedgingCall <HelloRequest, HelloReply>(CreateHedgingPolicy(serviceConfig.MethodConfigs[0].HedgingPolicy), invoker.Channel, ClientTestHelpers.ServiceMethod, new CallOptions()); // Act hedgingCall.StartUnary(new HelloRequest { Name = "World" }); Assert.IsFalse(hedgingCall.CreateHedgingCallsTask !.IsCompleted); // Assert Assert.AreEqual(1, hedgingCall._activeCalls.Count); await allCallsOnServerTcs.Task.DefaultTimeout(); Assert.AreEqual(5, callCount); Assert.AreEqual(5, hedgingCall._activeCalls.Count); hedgingCall.Dispose(); Assert.AreEqual(0, hedgingCall._activeCalls.Count); await hedgingCall.CreateHedgingCallsTask !.DefaultTimeout(); waitUntilFinishedTcs.SetResult(null); }
public async Task AsyncUnaryCall_ExceedAttempts_Failure() { // Arrange var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var requestMessages = new List <HelloRequest>(); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { // All calls are in-progress at once. Interlocked.Increment(ref callCount); if (callCount == 5) { tcs.TrySetResult(null); } await tcs.Task; var requestContent = await request.Content !.ReadAsStreamAsync(); var requestMessage = await ReadRequestMessage(requestContent); lock (requestMessages) { requestMessages.Add(requestMessage !); } return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(5, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(5, requestMessages.Count); foreach (var requestMessage in requestMessages) { Assert.AreEqual("World", requestMessage.Name); } }
public async Task Duplex_DeadlineExceedDuringDelay_Failure() { var callCount = 0; Task DuplexDeadlineExceeded(IAsyncStreamReader <DataMessage> requestStream, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context) { callCount++; return(Task.FromException(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata { new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()) }))); } // Arrange var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <DataMessage, DataMessage>(DuplexDeadlineExceeded); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var deadlineTimeout = 500; var call = client.DuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(deadlineTimeout))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseStream.MoveNext(CancellationToken.None)).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.RequestStream.WriteAsync(new DataMessage())).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(1, callCount); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); }
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure() { // Arrange var testSink = new TestSink(); var services = new ServiceCollection(); services.AddLogging(b => { b.AddProvider(new TestLoggerProvider(testSink)); }); services.AddNUnitLogger(); var provider = services.BuildServiceProvider(); var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async(request, ct) => { // Ensure SendAsync call doesn't hang upon cancellation by gRPC client. using var registration = ct.Register(() => tcs.TrySetCanceled()); Interlocked.Increment(ref callCount); return(await tcs.Task); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest { Name = "World" }); // Assert Assert.AreEqual(1, callCount); var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited"); Assert.AreEqual("Call commited. Reason: DeadlineExceeded", write.State.ToString()); }
public async Task AsyncUnaryCall_OneAttempt_Success(int maxAttempts) { // Arrange var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); await tcs.Task; await request.Content !.CopyToAsync(new MemoryStream()); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: maxAttempts); var invoker = HttpClientCallInvokerFactory.Create( httpClient, serviceConfig: serviceConfig, configure: o => o.MaxRetryAttempts = maxAttempts); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => callCount == maxAttempts, "All calls made at once."); tcs.SetResult(null); var rs = await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual("Hello world", rs.Message); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); }
public async Task AsyncClientStreamingCall_CompleteAndWriteAfterResult_Error() { // Arrange var requestContent = new MemoryStream(); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); _ = request.Content !.ReadAsStreamAsync(); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions()); // Assert var responseMessage = await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual("Hello world", responseMessage.Message); requestContent.Seek(0, SeekOrigin.Begin); await call.RequestStream.CompleteAsync().DefaultTimeout(); var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest { Name = "1" })).DefaultTimeout(); Assert.AreEqual("Request stream has already been completed.", ex.Message); }
public async Task AsyncUnaryCall_PushbackDelay_PushbackDelayUpdatesNextCallDelay() { // Arrange var stopwatch = new Stopwatch(); var callIntervals = new List <long>(); var hedgingDelay = TimeSpan.FromSeconds(10); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callIntervals.Add(stopwatch.ElapsedMilliseconds); stopwatch.Restart(); Interlocked.Increment(ref callCount); await request.Content !.CopyToAsync(new MemoryStream()); var hedgingPushback = hedgingDelay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); if (callCount == 1) { hedgingPushback = "0"; } return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: hedgingPushback)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: hedgingDelay); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act stopwatch.Start(); var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => callIntervals.Count == 2, "Only two calls should be made.").DefaultTimeout(); // First call should happen immediately Assert.LessOrEqual(callIntervals[0], 100); // Second call should happen after delay Assert.LessOrEqual(callIntervals[1], hedgingDelay.TotalMilliseconds); }
public async Task Unary_DeadlineExceedDuringDelay_Failure() { var callCount = 0; Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata { new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()) }))); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( hedgingDelay: TimeSpan.FromSeconds(10), nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(300))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.AreEqual(1, callCount); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded"); }
public async Task AsyncUnaryCall_ExceedAttempts_PusbackDelay_Failure() { // Arrange var stopwatch = new Stopwatch(); var callIntervals = new List <long>(); var hedgeDelay = TimeSpan.FromMilliseconds(100); const int timerResolutionMs = 15 * 2; // Timer has a precision of about 15ms. Double it, just to be safe var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { callIntervals.Add(stopwatch.ElapsedMilliseconds); stopwatch.Restart(); Interlocked.Increment(ref callCount); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: hedgeDelay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture))); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: hedgeDelay); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act stopwatch.Start(); var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(2, callCount); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); // First call should happen immediately Assert.LessOrEqual(callIntervals[0], hedgeDelay.TotalMilliseconds); // Second call should happen after delay Assert.GreaterOrEqual(callIntervals[1], hedgeDelay.TotalMilliseconds - timerResolutionMs); }
public async Task Unary_DeadlineExceedAfterServerCall_Failure(int exceptedServerCallCount) { var callCount = 0; var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously); Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context) { callCount++; if (callCount < exceptedServerCallCount) { return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, "")))); } return(tcs.Task); } // Arrange var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> { StatusCode.DeadlineExceeded }); var channel = CreateChannel(serviceConfig: serviceConfig); var client = TestClientFactory.Create(channel, method); // Act var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(200))); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode); Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled")); }
public async Task AsyncUnaryCall_ManyAttemptsNoDelay_MarshallerCalledOnce() { // Arrange var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { Interlocked.Increment(ref callCount); await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); var marshallerCount = 0; var requestMarshaller = Marshallers.Create <HelloRequest>( r => { Interlocked.Increment(ref marshallerCount); return(r.ToByteArray()); }, data => HelloRequest.Parser.ParseFrom(data)); var method = ClientTestHelpers.GetServiceMethod(requestMarshaller: requestMarshaller); // Act var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(method, string.Empty, new CallOptions(), new HelloRequest { Name = "World" }); // Assert var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout(); Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode); Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode); Assert.AreEqual(5, callCount); Assert.AreEqual(1, marshallerCount); }
public async Task Duplex_ManyParallelRequests_MessageRoundTripped() { const string ImportantMessage = @" _____ _____ _____ | __ \| __ \ / ____| __ _| |__) | |__) | | / _` | _ /| ___/| | | (_| | | \ \| | | |____ \__, |_| \_\_| \_____| __/ | |___/ _ (_) _ ___ | / __| | \__ \ _ |_|___/ | | ___ ___ ___ | | / __/ _ \ / _ \| | | (_| (_) | (_) | | \___\___/ \___/|_| "; var attempts = 100; var allUploads = new List <string>(); var allCompletedTasks = new List <Task>(); var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); async Task MessageUpload( IAsyncStreamReader <StringValue> requestStream, IServerStreamWriter <StringValue> responseStream, ServerCallContext context) { // Receive chunks var chunks = new List <string>(); await foreach (var chunk in requestStream.ReadAllAsync()) { chunks.Add(chunk.Value); } Task completeTask; lock (allUploads) { allUploads.Add(string.Join(Environment.NewLine, chunks)); if (allUploads.Count < attempts) { // Check that unused calls are canceled. completeTask = Task.Run(async() => { await tcs.Task; var cancellationTcs = new TaskCompletionSource <bool>(); context.CancellationToken.Register(s => ((TaskCompletionSource <bool>)s !).SetResult(true), cancellationTcs); await cancellationTcs.Task; }); } else { // Write response in used call. completeTask = Task.Run(async() => { // Write chunks foreach (var chunk in chunks) { await responseStream.WriteAsync(new StringValue { Value = chunk }); } }); } } await completeTask; } var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <StringValue, StringValue>(MessageUpload); var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 100, hedgingDelay: TimeSpan.Zero), maxRetryAttempts: 100); var client = TestClientFactory.Create(channel, method); using var call = client.DuplexStreamingCall(); var lines = ImportantMessage.Split(Environment.NewLine); for (var i = 0; i < lines.Length; i++) { await call.RequestStream.WriteAsync(new StringValue { Value = lines[i] }).DefaultTimeout(); await Task.Delay(TimeSpan.FromSeconds(0.01)).DefaultTimeout(); } await call.RequestStream.CompleteAsync().DefaultTimeout(); await TestHelpers.AssertIsTrueRetryAsync(() => allUploads.Count == 100, "Wait for all calls to reach server.").DefaultTimeout(); tcs.SetResult(null); var receivedLines = new List <string>(); await foreach (var line in call.ResponseStream.ReadAllAsync().DefaultTimeout()) { receivedLines.Add(line.Value); } Assert.AreEqual(ImportantMessage, string.Join(Environment.NewLine, receivedLines)); foreach (var upload in allUploads) { Assert.AreEqual(ImportantMessage, upload); } await Task.WhenAll(allCompletedTasks).DefaultTimeout(); }
public async Task ClientStreaming_WriteLargeMessageCausingCommit_Success() { SetExpectedErrorsFilter(writeContext => { return(true); }); var firstMessageTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var clientCancellationTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var serverCanceledTcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); const int maxAttempts = 2; var callCount = 0; var serverSuccessCount = 0; var serverAbortCount = 0; async Task <DataMessage> ClientStreamingWithCancellation(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { Logger.LogInformation("Server reading first message."); Assert.IsTrue(await requestStream.MoveNext()); if (Interlocked.Increment(ref callCount) >= maxAttempts) { firstMessageTcs.TrySetResult(null); } try { await requestStream.MoveNext(); Interlocked.Increment(ref serverSuccessCount); return(requestStream.Current); } catch (Exception ex) { Interlocked.Increment(ref serverAbortCount); serverCanceledTcs.TrySetException(ex); throw; } } // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithCancellation); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( maxAttempts: maxAttempts); var channel = CreateChannel(serviceConfig: serviceConfig, maxReceiveMessageSize: BigMessageSize * 2); var client = TestClientFactory.Create(channel, method); // Act var call = client.ClientStreamingCall(); // Assert Logger.LogInformation("Client sending first message."); await call.RequestStream.WriteAsync( new DataMessage { Data = ByteString.CopyFrom(Encoding.UTF8.GetBytes("Hello world")) }, CancellationToken.None).DefaultTimeout(); await firstMessageTcs.Task.DefaultTimeout(); // This large message causes the call to be commited and cancels other calls. Logger.LogInformation("Client sending large message."); var writeLargeMessageTask = call.RequestStream.WriteAsync( new DataMessage { Data = ByteString.CopyFrom(new byte[BigMessageSize]) }); var response = await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual(BigMessageSize, response.Data.Length); await ExceptionAssert.ThrowsAsync <Exception>(() => serverCanceledTcs.Task).DefaultTimeout(); Assert.AreEqual(1, serverSuccessCount); Assert.AreEqual(1, serverAbortCount); }
public async Task ClientStreaming_WriteAsyncCancellationDuring_ClientAbort() { SetExpectedErrorsFilter(writeContext => { return(true); }); var firstMessageTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var clientCancellationTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously); var serverCanceledTcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously); var maxAttempts = 2; var callCount = 0; async Task <DataMessage> ClientStreamingWithCancellation(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { Logger.LogInformation("Server reading first message."); Assert.IsTrue(await requestStream.MoveNext()); if (Interlocked.Increment(ref callCount) >= maxAttempts) { firstMessageTcs.TrySetResult(null); } Logger.LogInformation("Server waiting for canceled client message."); await clientCancellationTcs.Task; try { await requestStream.MoveNext(); throw new Exception("Should never reached here."); } catch (Exception ex) { if (IsWriteCanceledException(ex)) { serverCanceledTcs.SetResult(context.CancellationToken.IsCancellationRequested); return(new DataMessage()); } serverCanceledTcs.TrySetException(ex); throw; } } // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithCancellation); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig( maxAttempts: maxAttempts); var channel = CreateChannel( serviceConfig: serviceConfig, maxReceiveMessageSize: BigMessageSize * 2); var client = TestClientFactory.Create(channel, method); // Act var call = client.ClientStreamingCall(); // Assert Logger.LogInformation("Client sending first message."); await call.RequestStream.WriteAsync( new DataMessage { Data = ByteString.CopyFrom(Encoding.UTF8.GetBytes("Hello world")) }, CancellationToken.None).DefaultTimeout(); await firstMessageTcs.Task.DefaultTimeout(); Logger.LogInformation("Client sending large message."); var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); var clientEx = await ExceptionAssert.ThrowsAsync <RpcException>( () => call.RequestStream.WriteAsync( new DataMessage { Data = ByteString.CopyFrom(new byte[BigMessageSize]) }, cts.Token)).DefaultTimeout(); Assert.AreEqual(StatusCode.Cancelled, clientEx.StatusCode); clientCancellationTcs.SetResult(null); Assert.IsTrue(await serverCanceledTcs.Task.DefaultTimeout()); }
public async Task ClientStreaming_WriteAsyncFailsUntilRetries_WriteAsyncAwaitsUntilSuccess() { Task?largeWriteTask = null; var callCount = 0; bool?clientWriteWaitedForServerRead = null; async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context) { Logger.LogInformation("Server reading message 1."); Assert.IsTrue(await requestStream.MoveNext()); var currentCallCount = Interlocked.Increment(ref callCount); Logger.LogInformation("Server current call count: " + currentCallCount); if (currentCallCount <= 2) { Logger.LogInformation("Server pausing."); await Task.Delay(TimeSpan.FromMilliseconds(500)); Logger.LogInformation("Server erroring."); throw new RpcException(new Status(StatusCode.Unavailable, string.Empty)); } else { clientWriteWaitedForServerRead = !largeWriteTask !.IsCompleted; Logger.LogInformation("Server reading message 2."); Assert.IsTrue(await requestStream.MoveNext()); Logger.LogInformation("Server sending response."); return(new DataMessage()); } } SetExpectedErrorsFilter(writeContext => { return(true); }); // Arrange var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures); var channel = CreateChannel( serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMinutes(20)), maxReceiveMessageSize: BigMessageSize * 2, maxRetryBufferPerCallSize: BigMessageSize * 2); var client = TestClientFactory.Create(channel, method); // Act var call = client.ClientStreamingCall(); Logger.LogInformation("Client writing message 1."); await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }).DefaultTimeout(); Logger.LogInformation("Client writing message 2."); largeWriteTask = call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[BigMessageSize]) }); await largeWriteTask.DefaultTimeout(); // Assert Logger.LogInformation("Client waiting for response."); var response = await call.ResponseAsync.DefaultTimeout(); Assert.IsTrue(clientWriteWaitedForServerRead); }
public async Task AsyncClientStreamingCall_SuccessAfterRetry_RequestContentSent(int hedgingDelayMS) { // Arrange var callLock = new object(); var requestContent = new MemoryStream(); var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { var firstCall = false; lock (callLock) { callCount++; if (callCount == 1) { firstCall = true; } } if (firstCall) { await request.Content !.CopyToAsync(new MemoryStream()); return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable)); } var content = (PushStreamContent <HelloRequest, HelloReply>)request.Content !; await content.PushComplete.DefaultTimeout(); await request.Content !.CopyToAsync(requestContent); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMS)); var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig); // Act var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions()); // Assert Assert.IsNotNull(call); var responseTask = call.ResponseAsync; Assert.IsFalse(responseTask.IsCompleted, "Response not returned until client stream is complete."); await call.RequestStream.WriteAsync(new HelloRequest { Name = "1" }).DefaultTimeout(); await call.RequestStream.WriteAsync(new HelloRequest { Name = "2" }).DefaultTimeout(); await call.RequestStream.CompleteAsync().DefaultTimeout(); var responseMessage = await responseTask.DefaultTimeout(); Assert.AreEqual("Hello world", responseMessage.Message); requestContent.Seek(0, SeekOrigin.Begin); var requests = new List <HelloRequest>(); while (true) { var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout(); if (requestMessage == null) { break; } requests.Add(requestMessage); } Assert.AreEqual(2, requests.Count); Assert.AreEqual("1", requests[0].Name); Assert.AreEqual("2", requests[1].Name); }
public async Task AsyncClientStreamingCall_ManyParallelCalls_ReadDirectlyToRequestStream() { // Arrange var requestStreams = new List <WriterTestStream>(); var attempts = 100; var callCount = 0; var httpClient = ClientTestHelpers.CreateTestClient(async request => { WriterTestStream writerTestStream; lock (requestStreams) { Interlocked.Increment(ref callCount); writerTestStream = new WriterTestStream(); requestStreams.Add(writerTestStream); } await request.Content !.CopyToAsync(writerTestStream); var reply = new HelloReply { Message = "Hello world" }; var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout(); return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)); }); var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: attempts); var invoker = HttpClientCallInvokerFactory.Create( httpClient, serviceConfig: serviceConfig, configure: o => o.MaxRetryAttempts = attempts); // Act var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions()); var writeAsyncTask = call.RequestStream.WriteAsync(new HelloRequest { Name = "World" }); // Assert await TestHelpers.AssertIsTrueRetryAsync(() => callCount == attempts, "All calls made at once."); var firstMessages = await Task.WhenAll(requestStreams.Select(s => s.WaitForDataAsync())).DefaultTimeout(); await writeAsyncTask.DefaultTimeout(); foreach (var message in firstMessages) { Assert.IsTrue(firstMessages[0].Span.SequenceEqual(message.Span)); } writeAsyncTask = call.RequestStream.WriteAsync(new HelloRequest { Name = "World 2" }); var secondMessages = await Task.WhenAll(requestStreams.Select(s => s.WaitForDataAsync())).DefaultTimeout(); await writeAsyncTask.DefaultTimeout(); foreach (var message in secondMessages) { Assert.IsTrue(secondMessages[0].Span.SequenceEqual(message.Span)); } await call.RequestStream.CompleteAsync().DefaultTimeout(); var rs = await call.ResponseAsync.DefaultTimeout(); Assert.AreEqual("Hello world", rs.Message); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); }