public async Task AsyncUnaryCall_DisposeDuringBackoff_CanceledStatus()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
var cts = new CancellationTokenSource();
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(cancellationToken: cts.Token), new HelloRequest {
Name = "World"
});
var delayTask = Task.Delay(100);
var completedTask = await Task.WhenAny(call.ResponseAsync, delayTask);
// Assert
Assert.AreEqual(delayTask, completedTask); // Ensure that we're waiting for retry
call.Dispose();
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
Assert.AreEqual("gRPC call disposed.", ex.Status.Detail);
}
public async Task AsyncUnaryCall_Success_RequestContentSent()
{
// Arrange
HttpContent?content = null;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
content = request.Content;
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message);
}
public async Task AsyncUnaryCall_FailureWithLongDelay_Dispose_CallImmediatelyDisposed()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
});
// Very long delay
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(initialBackoff: TimeSpan.FromSeconds(30), maxBackoff: TimeSpan.FromSeconds(30));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var resultTask = call.ResponseAsync;
// Test will timeout if dispose doesn't kill the timer.
call.Dispose();
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => resultTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
Assert.AreEqual("gRPC call disposed.", ex.Status.Detail);
}
public async Task ClientStreamWriter_WriteWhilePendingWrite_ErrorThrown()
{
// Arrange
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
var streamContent = new StreamContent(new SyncPointMemoryStream());
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
var writeTask1 = call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
});
Assert.IsFalse(writeTask1.IsCompleted);
var writeTask2 = call.RequestStream.WriteAsync(new HelloRequest {
Name = "2"
});
var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => writeTask2).DefaultTimeout();
Assert.AreEqual("Can't write the message because the previous write is in progress.", ex.Message);
}
public async Task AsyncUnaryCall_SuccessAfterRetry_AccessResponseHeaders_SuccessfullyResponseHeadersReturned()
{
// Arrange
HttpContent?content = null;
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
content = request.Content !;
if (callCount == 1)
{
await content.CopyToAsync(new MemoryStream());
await syncPoint.WaitForSyncPoint();
return(ResponseUtils.CreateHeadersOnlyResponse(
HttpStatusCode.OK,
StatusCode.Unavailable,
customHeaders: new Dictionary <string, string> {
["call-count"] = callCount.ToString()
}));
}
syncPoint.Continue();
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(
HttpStatusCode.OK,
streamContent,
customHeaders: new Dictionary <string, string> {
["call-count"] = callCount.ToString()
}));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var headersTask = call.ResponseHeadersAsync;
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
// Assert
Assert.AreEqual(2, callCount);
Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message);
var headers = await headersTask.DefaultTimeout();
Assert.AreEqual("2", headers.GetValue("call-count"));
}
public async Task Unary_TriggerRetryThrottling_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(
retryThrottling: new RetryThrottlingPolicy
{
MaxTokens = 5,
TokenRatio = 0.1
}));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 3, Retry: False");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Throttled");
}
public async Task Unary_AttemptsGreaterThanDefaultClientLimit_LimitedAttemptsMade(int hedgingDelay)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(maxAttempts: 10, initialBackoff: TimeSpan.FromMilliseconds(hedgingDelay)));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, callCount);
AssertHasLog(LogLevel.Debug, "MaxAttemptsLimited", "The method has 10 attempts specified in the service config. The number of attempts has been limited by channel configuration to 5.");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task AsyncUnaryCall_UnsupportedStatusCode_Failure()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StringContent(""), StatusCode.InvalidArgument));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode);
Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode);
}
public async Task Unary_ExceedRetryAttempts_Failure()
{
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
var metadata = new Metadata();
metadata.Add("grpc-retry-pushback-ms", "5");
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""), metadata)));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig());
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "RetryPushbackReceived", "Retry pushback of '5' received from the failed gRPC call.");
AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 1, Retry: True");
AssertHasLog(LogLevel.Trace, "StartingRetryDelay", "Starting retry delay of 00:00:00.0050000.");
AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 5, Retry: False");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task AsyncUnaryCall_AuthInteceptorDispose_Error()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var credentialsSyncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var credentials = CallCredentials.FromInterceptor(async(context, metadata) =>
{
await credentialsSyncPoint.WaitToContinue();
metadata.Add("Authorization", $"Bearer TEST");
});
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig, configure: options => options.Credentials = ChannelCredentials.Create(new SslCredentials(), credentials));
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var responseTask = call.ResponseAsync;
var responseHeadersTask = call.ResponseHeadersAsync;
await credentialsSyncPoint.WaitForSyncPoint().DefaultTimeout();
call.Dispose();
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseHeadersTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: Canceled", write.State.ToString());
}
public async Task ClientStreaming_MultipleWritesAndRetries_Failure()
{
var nextFailure = 1;
async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
List <byte> bytes = new List <byte>();
await foreach (var message in requestStream.ReadAllAsync())
{
if (bytes.Count >= nextFailure)
{
nextFailure = nextFailure * 2;
throw new RpcException(new Status(StatusCode.Unavailable, ""));
}
bytes.Add(message.Data[0]);
}
return(new DataMessage
{
Data = ByteString.CopyFrom(bytes.ToArray())
});
}
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(maxAttempts: 10), maxRetryAttempts: 10);
var client = TestClientFactory.Create(channel, method);
var sentData = new List <byte>();
// Act
var call = client.ClientStreamingCall();
for (var i = 0; i < 20; i++)
{
sentData.Add((byte)i);
await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)i }) }).DefaultTimeout();
await Task.Delay(1);
}
await call.RequestStream.CompleteAsync().DefaultTimeout();
var result = await call.ResponseAsync.DefaultTimeout();
// Assert
Assert.IsTrue(result.Data.Span.SequenceEqual(sentData.ToArray()));
}
public async Task AsyncServerStreamingCall_SuccessAfterRetry_RequestContentSent()
{
// Arrange
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
var content = request.Content !;
await content.CopyToAsync(requestContent);
requestContent.Seek(0, SeekOrigin.Begin);
if (callCount == 1)
{
await syncPoint.WaitForSyncPoint();
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
syncPoint.Continue();
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None);
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
// Assert
Assert.IsTrue(await moveNextTask);
Assert.AreEqual("Hello world", call.ResponseStream.Current.Message);
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("World", requestMessage !.Name);
requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.IsNull(requestMessage);
}
public async Task AsyncServerStreamingCall_FailureAfterReadingResponseMessage_Failure()
{
// Arrange
var streamContent = new SyncPointMemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
callCount++;
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent))));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest());
var responseStream = call.ResponseStream;
// Assert
Assert.IsNull(responseStream.Current);
var moveNextTask1 = responseStream.MoveNext(CancellationToken.None);
Assert.IsFalse(moveNextTask1.IsCompleted);
await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply
{
Message = "Hello world 1"
}).DefaultTimeout()).DefaultTimeout();
Assert.IsTrue(await moveNextTask1.DefaultTimeout());
Assert.IsNotNull(responseStream.Current);
Assert.AreEqual("Hello world 1", responseStream.Current.Message);
var moveNextTask2 = responseStream.MoveNext(CancellationToken.None);
Assert.IsFalse(moveNextTask2.IsCompleted);
await streamContent.AddExceptionAndWait(new Exception("Exception!")).DefaultTimeout();
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => moveNextTask2).DefaultTimeout();
Assert.AreEqual(StatusCode.Internal, ex.StatusCode);
Assert.AreEqual(StatusCode.Internal, call.GetStatus().StatusCode);
Assert.AreEqual("Error reading next message. Exception: Exception!", call.GetStatus().Detail);
}
public async Task Unary_MultipleLargeMessages_ExceedChannelMaxBufferSize()
{
// Arrange
var sp1 = new SyncPoint(runContinuationsAsynchronously: true);
var sp2 = new SyncPoint(runContinuationsAsynchronously: true);
var sp3 = new SyncPoint(runContinuationsAsynchronously: true);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(),
maxRetryBufferSize: 200,
maxRetryBufferPerCallSize: 100);
var request = new DataMessage {
Data = ByteString.CopyFrom(new byte[90])
};
// Act
var call1Task = MakeCall(Fixture, channel, request, sp1);
await sp1.WaitForSyncPoint();
var call2Task = MakeCall(Fixture, channel, request, sp2);
await sp2.WaitForSyncPoint();
// Will exceed channel buffer limit and won't retry
var call3Task = MakeCall(Fixture, channel, request, sp3);
await sp3.WaitForSyncPoint();
// Assert
Assert.AreEqual(194, channel.CurrentRetryBufferSize);
sp1.Continue();
sp2.Continue();
sp3.Continue();
var response = await call1Task.DefaultTimeout();
Assert.AreEqual(90, response.Data.Length);
response = await call2Task.DefaultTimeout();
Assert.AreEqual(90, response.Data.Length);
// Can't retry because buffer size exceeded.
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call3Task).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
public async Task ServerStreaming_CancellatonTokenSpecified_TokenUnregisteredAndResourcesReleased()
{
Task FakeServerStreamCall(DataMessage request, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context)
{
return(Task.CompletedTask);
}
// Arrange
var method = Fixture.DynamicGrpc.AddServerStreamingMethod <DataMessage, DataMessage>(FakeServerStreamCall);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var references = new List <WeakReference>();
// Checking that token register calls don't build up on CTS and create a memory leak.
var cts = new CancellationTokenSource();
// Act
// Send calls in a different method so there is no chance that a stack reference
// to a gRPC call is still alive after calls are complete.
await MakeCallsAsync(channel, method, references, cts.Token).DefaultTimeout();
// Assert
// There is a race when cleaning up cancellation token registry.
// Retry a few times to ensure GC is run after unregister.
await TestHelpers.AssertIsTrueRetryAsync(() =>
{
GC.Collect();
GC.WaitForPendingFinalizers();
for (var i = 0; i < references.Count; i++)
{
if (references[i].IsAlive)
{
return(false);
}
}
// Resources for past calls were successfully GCed.
return(true);
}, "Assert that retry call resources are released.");
}
public async Task Duplex_DeadlineExceedDuringBackoff_Failure()
{
var callCount = 0;
Task DuplexDeadlineExceeded(IAsyncStreamReader <DataMessage> requestStream, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context)
{
callCount++;
return(Task.FromException(new RpcException(new Status(StatusCode.Unavailable, ""), new Metadata
{
new Metadata.Entry("grpc-retry-pushback-ms", TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <DataMessage, DataMessage>(DuplexDeadlineExceeded);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
retryableStatusCodes: new List <StatusCode> {
StatusCode.Unavailable
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.DuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(300)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseStream.MoveNext(CancellationToken.None)).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.RequestStream.WriteAsync(new DataMessage())).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
}
public async Task Unary_DeadlineExceedAfterServerCall_Failure(int exceptedServerCallCount)
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
if (callCount < exceptedServerCallCount)
{
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""))));
}
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(200)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(exceptedServerCallCount, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
tcs.SetResult(new DataMessage());
}
public async Task ClientStreamWriter_WriteWhileComplete_ErrorThrown()
{
// Arrange
var streamContent = new SyncPointMemoryStream();
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent))));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
await call.RequestStream.CompleteAsync().DefaultTimeout();
var resultTask = call.ResponseAsync;
// Assert
var writeException1 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", writeException1.Message);
await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply
{
Message = "Hello world 1"
}).DefaultTimeout()).DefaultTimeout();
await streamContent.AddDataAndWait(new byte[0]);
var result = await resultTask.DefaultTimeout();
Assert.AreEqual("Hello world 1", result.Message);
var writeException2 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "2"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", writeException2.Message);
}
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig());
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage
{
Data = ByteString.CopyFrom(new byte[payloadSize])
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
if (!exceedBufferLimit)
{
Assert.AreEqual(5, callCount);
}
else
{
Assert.AreEqual(1, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded");
}
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
}
public async Task Unary_DeadlineExceedDuringBackoff_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailureWithPushback(DataMessage request, ServerCallContext context)
{
callCount++;
Logger.LogInformation($"Server sending pushback for call {callCount}.");
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""), new Metadata
{
new Metadata.Entry("grpc-retry-pushback-ms", TimeSpan.FromSeconds(10).TotalMilliseconds.ToString(CultureInfo.InvariantCulture))
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailureWithPushback, nameof(UnaryFailureWithPushback));
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(
initialBackoff: TimeSpan.FromSeconds(10),
maxBackoff: TimeSpan.FromSeconds(10),
retryableStatusCodes: new List <StatusCode> {
StatusCode.Unavailable
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(500)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
}
public async Task AsyncClientStreamingCall_CompleteAndWriteAfterResult_Error()
{
// Arrange
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
_ = request.Content !.ReadAsStreamAsync();
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
var responseMessage = await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual("Hello world", responseMessage.Message);
requestContent.Seek(0, SeekOrigin.Begin);
await call.RequestStream.CompleteAsync().DefaultTimeout();
var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", ex.Message);
}
public async Task AsyncUnaryCall_Success_SuccussCommitLogged()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var content = request.Content !;
await content.CopyToAsync(new MemoryStream());
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.Unary), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
await call.ResponseAsync.DefaultTimeout();
// Assert
var log = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: ResponseHeadersReceived", log.State.ToString());
}
public async Task AsyncUnaryCall_PushbackExpicitDelay_DelayForSpecifiedDuration()
{
// Arrange
Task?delayTask = null;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
if (callCount == 1)
{
await request.Content !.CopyToAsync(new MemoryStream());
delayTask = Task.Delay(100);
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: "200"));
}
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(backoffMultiplier: 1);
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Delay of 100ms will finish before second record which has a pushback delay of 200ms
var completedTask = await Task.WhenAny(call.ResponseAsync, delayTask !).DefaultTimeout();
var rs = await call.ResponseAsync.DefaultTimeout();
// Assert
Assert.AreEqual(delayTask, completedTask); // Response task should finish after
Assert.AreEqual(2, callCount);
Assert.AreEqual("Hello world", rs.Message);
}
public async Task Unary_CanceledBeforeServerCall_Failure()
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
var cts = new CancellationTokenSource();
cts.Cancel();
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(cancellationToken: cts.Token));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
Assert.AreEqual(StatusCode.Cancelled, call.GetStatus().StatusCode);
Assert.AreEqual(0, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Canceled");
tcs.SetResult(new DataMessage());
}
public async Task AsyncUnaryCall_NoMessagesSuccess_Failure()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var content = request.Content !;
await content.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.OK));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.Unary), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
// Assert
Assert.AreEqual("Failed to deserialize response message.", ex.Status.Detail);
Assert.AreEqual(StatusCode.Internal, ex.StatusCode);
var log = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: FatalStatusCode", log.State.ToString());
}
public async Task AsyncUnaryCall_PushbackExplicitDelayExceedAttempts_Failure()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: "0"));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(maxAttempts: 5);
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(5, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
}
public async Task AsyncServerStreamingCall_NoMessagesSuccess_SuccussCommitLogged()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var content = request.Content !;
await content.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.OK));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None);
// Assert
Assert.IsFalse(await moveNextTask);
var log = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: ResponseHeadersReceived", log.State.ToString());
}
public async Task AsyncUnaryCall_SuccessAfterRetry_RequestContentSent()
{
// Arrange
HttpContent?content = null;
bool? firstRequestPreviousAttemptsHeader = null;
string?secondRequestPreviousAttemptsHeaderValue = null;
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
content = request.Content !;
await content.CopyToAsync(requestContent);
requestContent.Seek(0, SeekOrigin.Begin);
if (callCount == 1)
{
firstRequestPreviousAttemptsHeader = request.Headers.TryGetValues(GrpcProtocolConstants.RetryPreviousAttemptsHeader, out _);
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
if (request.Headers.TryGetValues(GrpcProtocolConstants.RetryPreviousAttemptsHeader, out var retryAttemptCountValue))
{
secondRequestPreviousAttemptsHeaderValue = retryAttemptCountValue.Single();
}
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent, customTrailers: new Dictionary <string, string>
{
["custom-trailer"] = "Value!"
}));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(2, callCount);
Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message);
Assert.AreEqual("1", (await call.ResponseHeadersAsync.DefaultTimeout()).GetValue(GrpcProtocolConstants.RetryPreviousAttemptsHeader));
Assert.IsNotNull(content);
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("World", requestMessage !.Name);
Assert.IsFalse(firstRequestPreviousAttemptsHeader);
Assert.AreEqual("1", secondRequestPreviousAttemptsHeaderValue);
var trailers = call.GetTrailers();
Assert.AreEqual("Value!", trailers.GetValue("custom-trailer"));
}
public async Task AsyncClientStreamingCall_SuccessAfterRetry_RequestContentSent()
{
// Arrange
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
var currentContent = new MemoryStream();
await request.Content !.CopyToAsync(currentContent);
if (callCount == 1)
{
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
currentContent.Seek(0, SeekOrigin.Begin);
await currentContent.CopyToAsync(requestContent);
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
Assert.IsNotNull(call);
var responseTask = call.ResponseAsync;
Assert.IsFalse(responseTask.IsCompleted, "Response not returned until client stream is complete.");
await call.RequestStream.WriteAsync(new HelloRequest { Name = "1" }).DefaultTimeout();
await call.RequestStream.WriteAsync(new HelloRequest { Name = "2" }).DefaultTimeout();
await call.RequestStream.CompleteAsync().DefaultTimeout();
var responseMessage = await responseTask.DefaultTimeout();
Assert.AreEqual("Hello world", responseMessage.Message);
requestContent.Seek(0, SeekOrigin.Begin);
var requests = new List <HelloRequest>();
while (true)
{
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
if (requestMessage == null)
{
break;
}
requests.Add(requestMessage);
}
Assert.AreEqual(2, requests.Count);
Assert.AreEqual("1", requests[0].Name);
Assert.AreEqual("2", requests[1].Name);
call.Dispose();
}
public async Task AsyncClientStreamingCall_OneMessageSentThenRetryThenAnotherMessage_RequestContentSent()
{
// Arrange
var requestContent = new MemoryStream();
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
var content = (PushStreamContent <HelloRequest, HelloReply>)request.Content !;
if (callCount == 1)
{
_ = content.CopyToAsync(new MemoryStream());
await syncPoint.WaitForSyncPoint();
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
syncPoint.Continue();
await content.PushComplete.DefaultTimeout();
await content.CopyToAsync(requestContent);
requestContent.Seek(0, SeekOrigin.Begin);
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
Assert.IsNotNull(call);
var responseTask = call.ResponseAsync;
Assert.IsFalse(responseTask.IsCompleted, "Response not returned until client stream is complete.");
await call.RequestStream.WriteAsync(new HelloRequest { Name = "1" }).DefaultTimeout();
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
await call.RequestStream.WriteAsync(new HelloRequest { Name = "2" }).DefaultTimeout();
await call.RequestStream.CompleteAsync().DefaultTimeout();
var responseMessage = await responseTask.DefaultTimeout();
Assert.AreEqual("Hello world", responseMessage.Message);
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("1", requestMessage !.Name);
requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("2", requestMessage !.Name);
requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.IsNull(requestMessage);
}
