public async Task AsyncUnaryCall_FatalStatusCode_HedgeDelay_Failure()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, (callCount == 1) ? StatusCode.Unavailable : StatusCode.InvalidArgument));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(50));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode);
Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode);
Assert.AreEqual(2, callCount);
}
public async Task Unary_AttemptsGreaterThanDefaultClientLimit_LimitedAttemptsMade(int hedgingDelay)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 10, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelay)));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, callCount);
AssertHasLog(LogLevel.Debug, "MaxAttemptsLimited", "The method has 10 attempts specified in the service config. The number of attempts has been limited by channel configuration to 5.");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task Unary_ExceedRetryAttempts_Failure()
{
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
var metadata = new Metadata();
metadata.Add("grpc-retry-pushback-ms", "5");
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""), metadata)));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig());
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "RetryPushbackReceived", "Retry pushback of '5' received from the failed gRPC call.");
AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 1, Retry: True");
AssertHasLog(LogLevel.Trace, "StartingRetryDelay", "Starting retry delay of 00:00:00.0050000.");
AssertHasLog(LogLevel.Debug, "RetryEvaluated", "Evaluated retry for failed gRPC call. Status code: 'Unavailable', Attempt: 5, Retry: False");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task Unary_ExceedAttempts_Failure(int?hedgingDelay)
{
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Ignore errors
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var delay = (hedgingDelay == null)
? (TimeSpan?)null
: TimeSpan.FromMilliseconds(hedgingDelay.Value);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: delay));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task Unary_DeadlineExceedBeforeServerCall_Failure()
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(0, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
tcs.SetResult(new DataMessage());
}
public async Task ClientStreamWriter_WriteWhilePendingWrite_ErrorThrown()
{
// Arrange
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
var streamContent = new StreamContent(new SyncPointMemoryStream());
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent)));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
var writeTask1 = call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
});
Assert.IsFalse(writeTask1.IsCompleted);
var writeTask2 = call.RequestStream.WriteAsync(new HelloRequest {
Name = "2"
});
var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => writeTask2).DefaultTimeout();
Assert.AreEqual("Can't write the message because the previous write is in progress.", ex.Message);
}
public async Task Unary_TriggerRetryThrottling_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
retryThrottling: new RetryThrottlingPolicy
{
MaxTokens = 5,
TokenRatio = 0.1
}));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(3, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Throttled");
}
public async Task AsyncUnaryCall_UnsupportedStatusCode_Failure()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StringContent(""), StatusCode.InvalidArgument));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode);
Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode);
}
public async Task AsyncUnaryCall_Success_RequestContentSent()
{
// Arrange
HttpContent?content = null;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
content = request.Content;
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message);
}
public async Task AsyncUnaryCall_DisposeDuringBackoff_CanceledStatus()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: TimeSpan.FromSeconds(10).TotalMilliseconds.ToString()));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
var cts = new CancellationTokenSource();
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(cancellationToken: cts.Token), new HelloRequest {
Name = "World"
});
var delayTask = Task.Delay(100);
var completedTask = await Task.WhenAny(call.ResponseAsync, delayTask);
// Assert
Assert.AreEqual(delayTask, completedTask); // Ensure that we're waiting for retry
call.Dispose();
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
Assert.AreEqual("gRPC call disposed.", ex.Status.Detail);
}
public async Task AsyncUnaryCall_FailureWithLongDelay_Dispose_CallImmediatelyDisposed()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
});
// Very long delay
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(initialBackoff: TimeSpan.FromSeconds(30), maxBackoff: TimeSpan.FromSeconds(30));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var resultTask = call.ResponseAsync;
// Test will timeout if dispose doesn't kill the timer.
call.Dispose();
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => resultTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
Assert.AreEqual("gRPC call disposed.", ex.Status.Detail);
}
public async Task AsyncUnaryCall_SuccessAfterRetry_AccessResponseHeaders_SuccessfullyResponseHeadersReturned()
{
// Arrange
HttpContent?content = null;
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callCount++;
content = request.Content !;
if (callCount == 1)
{
await content.CopyToAsync(new MemoryStream());
await syncPoint.WaitForSyncPoint();
return(ResponseUtils.CreateHeadersOnlyResponse(
HttpStatusCode.OK,
StatusCode.Unavailable,
customHeaders: new Dictionary <string, string> {
["call-count"] = callCount.ToString()
}));
}
syncPoint.Continue();
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(
HttpStatusCode.OK,
streamContent,
customHeaders: new Dictionary <string, string> {
["call-count"] = callCount.ToString()
}));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var headersTask = call.ResponseHeadersAsync;
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
// Assert
Assert.AreEqual(2, callCount);
Assert.AreEqual("Hello world", (await call.ResponseAsync.DefaultTimeout()).Message);
var headers = await headersTask.DefaultTimeout();
Assert.AreEqual("2", headers.GetValue("call-count"));
}
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure()
{
// Arrange
var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
Interlocked.Increment(ref callCount);
return(tcs.Task);
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
}
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit, int hedgingDelayMilliseconds)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Ignore errors
SetExpectedErrorsFilter(writeContext =>
{
if (writeContext.EventId.Name == "ErrorSendingMessage" ||
writeContext.EventId.Name == "ErrorExecutingServiceMethod")
{
return(true);
}
return(false);
});
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMilliseconds)),
maxReceiveMessageSize: (int)GrpcChannel.DefaultMaxRetryBufferPerCallSize * 2);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage
{
Data = ByteString.CopyFrom(new byte[payloadSize])
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
if (!exceedBufferLimit)
{
Assert.AreEqual(5, callCount);
}
else
{
Assert.AreEqual(1, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded");
// Cancelled calls could cause server errors. Delay so these error don't show up
// in the next unit test.
await Task.Delay(100);
}
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
}
public async Task ClientStreaming_WriteAsyncCancellationDuringRetry_Canceled(bool throwOperationCanceledOnCancellation)
{
async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
Logger.LogInformation("Server reading message 1.");
Assert.IsTrue(await requestStream.MoveNext());
Logger.LogInformation("Server pausing.");
await Task.Delay(TimeSpan.FromMilliseconds(500));
Logger.LogInformation("Server erroring.");
throw new RpcException(new Status(StatusCode.Unavailable, string.Empty));
}
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromSeconds(20)),
maxReceiveMessageSize: BigMessageSize * 2,
maxRetryBufferPerCallSize: BigMessageSize * 2,
throwOperationCanceledOnCancellation: throwOperationCanceledOnCancellation);
var client = TestClientFactory.Create(channel, method);
// Act
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var call = client.ClientStreamingCall();
Logger.LogInformation("Client writing message 1.");
await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }, cts.Token).DefaultTimeout();
Logger.LogInformation("Client writing message 2.");
var writeTask = call.RequestStream.WriteAsync(new DataMessage {
Data = ByteString.CopyFrom(new byte[BigMessageSize])
}, cts.Token);
// Assert
if (throwOperationCanceledOnCancellation)
{
var ex = await ExceptionAssert.ThrowsAsync <OperationCanceledException>(() => writeTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, call.GetStatus().StatusCode);
}
else
{
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => writeTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
}
Assert.IsTrue(cts.Token.IsCancellationRequested, "WriteAsync finished when CancellationToken wasn't triggered.");
}
public async Task ClientStreaming_MultipleWritesAndRetries_Failure()
{
var nextFailure = 1;
async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
List <byte> bytes = new List <byte>();
await foreach (var message in requestStream.ReadAllAsync())
{
if (bytes.Count >= nextFailure)
{
nextFailure = nextFailure * 2;
throw new RpcException(new Status(StatusCode.Unavailable, ""));
}
bytes.Add(message.Data[0]);
}
return(new DataMessage
{
Data = ByteString.CopyFrom(bytes.ToArray())
});
}
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(maxAttempts: 10), maxRetryAttempts: 10);
var client = TestClientFactory.Create(channel, method);
var sentData = new List <byte>();
// Act
var call = client.ClientStreamingCall();
for (var i = 0; i < 20; i++)
{
sentData.Add((byte)i);
await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)i }) }).DefaultTimeout();
await Task.Delay(1);
}
await call.RequestStream.CompleteAsync().DefaultTimeout();
var result = await call.ResponseAsync.DefaultTimeout();
// Assert
Assert.IsTrue(result.Data.Span.SequenceEqual(sentData.ToArray()));
}
public async Task AsyncServerStreamingCall_SuccessAfterRetry_RequestContentSent()
{
// Arrange
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
MemoryStream?requestContent = null;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
var s = await request.Content !.ReadAsStreamAsync();
var ms = new MemoryStream();
await s.CopyToAsync(ms);
if (callCount == 1)
{
await syncPoint.WaitForSyncPoint();
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
syncPoint.Continue();
requestContent = ms;
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(50));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None);
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
// Assert
Assert.IsTrue(await moveNextTask);
Assert.AreEqual("Hello world", call.ResponseStream.Current.Message);
requestContent !.Seek(0, SeekOrigin.Begin);
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("World", requestMessage !.Name);
}
public async Task AsyncUnaryCall_AuthInteceptorDispose_Error()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var credentialsSyncPoint = new SyncPoint(runContinuationsAsynchronously: true);
var credentials = CallCredentials.FromInterceptor(async(context, metadata) =>
{
await credentialsSyncPoint.WaitToContinue();
metadata.Add("Authorization", $"Bearer TEST");
});
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig, configure: options => options.Credentials = ChannelCredentials.Create(new SslCredentials(), credentials));
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var responseTask = call.ResponseAsync;
var responseHeadersTask = call.ResponseHeadersAsync;
await credentialsSyncPoint.WaitForSyncPoint().DefaultTimeout();
call.Dispose();
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => responseHeadersTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: Canceled", write.State.ToString());
}
public async Task Unary_RetryThrottlingBecomesActive_HasDelay_Failure()
{
var callCount = 0;
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
async Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
await syncPoint.WaitToContinue();
return(request);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromMilliseconds(100),
retryThrottling: new RetryThrottlingPolicy
{
MaxTokens = 5,
TokenRatio = 0.1
}));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
await syncPoint.WaitForSyncPoint().DefaultTimeout();
// Manually trigger retry throttling
Debug.Assert(channel.RetryThrottling != null);
channel.RetryThrottling.CallFailure();
channel.RetryThrottling.CallFailure();
channel.RetryThrottling.CallFailure();
Debug.Assert(channel.RetryThrottling.IsRetryThrottlingActive());
// Assert
await TestHelpers.AssertIsTrueRetryAsync(() => HasLog(LogLevel.Debug, "AdditionalCallsBlockedByRetryThrottling", "Additional calls blocked by retry throttling."), "Check for expected log.");
Assert.AreEqual(1, callCount);
syncPoint.Continue();
await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ResponseHeadersReceived");
}
public async Task Dispose_ActiveCalls_CleansUpActiveCalls()
{
// Arrange
var allCallsOnServerTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var waitUntilFinishedTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
// All calls are in-progress at once.
Interlocked.Increment(ref callCount);
if (callCount == 5)
{
allCallsOnServerTcs.SetResult(null);
}
await waitUntilFinishedTcs.Task;
await request.Content !.CopyToAsync(new MemoryStream());
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMilliseconds(20));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
var hedgingCall = new HedgingCall <HelloRequest, HelloReply>(CreateHedgingPolicy(serviceConfig.MethodConfigs[0].HedgingPolicy), invoker.Channel, ClientTestHelpers.ServiceMethod, new CallOptions());
// Act
hedgingCall.StartUnary(new HelloRequest {
Name = "World"
});
Assert.IsFalse(hedgingCall.CreateHedgingCallsTask !.IsCompleted);
// Assert
Assert.AreEqual(1, hedgingCall._activeCalls.Count);
await allCallsOnServerTcs.Task.DefaultTimeout();
Assert.AreEqual(5, callCount);
Assert.AreEqual(5, hedgingCall._activeCalls.Count);
hedgingCall.Dispose();
Assert.AreEqual(0, hedgingCall._activeCalls.Count);
await hedgingCall.CreateHedgingCallsTask !.DefaultTimeout();
waitUntilFinishedTcs.SetResult(null);
}
public async Task AsyncServerStreamingCall_FailureAfterReadingResponseMessage_Failure()
{
// Arrange
var streamContent = new SyncPointMemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
callCount++;
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent))));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest());
var responseStream = call.ResponseStream;
// Assert
Assert.IsNull(responseStream.Current);
var moveNextTask1 = responseStream.MoveNext(CancellationToken.None);
Assert.IsFalse(moveNextTask1.IsCompleted);
await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply
{
Message = "Hello world 1"
}).DefaultTimeout()).DefaultTimeout();
Assert.IsTrue(await moveNextTask1.DefaultTimeout());
Assert.IsNotNull(responseStream.Current);
Assert.AreEqual("Hello world 1", responseStream.Current.Message);
var moveNextTask2 = responseStream.MoveNext(CancellationToken.None);
Assert.IsFalse(moveNextTask2.IsCompleted);
await streamContent.AddExceptionAndWait(new Exception("Exception!")).DefaultTimeout();
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => moveNextTask2).DefaultTimeout();
Assert.AreEqual(StatusCode.Internal, ex.StatusCode);
Assert.AreEqual(StatusCode.Internal, call.GetStatus().StatusCode);
Assert.AreEqual("Error reading next message. Exception: Exception!", call.GetStatus().Detail);
}
public async Task AsyncUnaryCall_ExceedAttempts_Failure()
{
// Arrange
var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var requestMessages = new List <HelloRequest>();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
// All calls are in-progress at once.
Interlocked.Increment(ref callCount);
if (callCount == 5)
{
tcs.TrySetResult(null);
}
await tcs.Task;
var requestContent = await request.Content !.ReadAsStreamAsync();
var requestMessage = await ReadRequestMessage(requestContent);
lock (requestMessages)
{
requestMessages.Add(requestMessage !);
}
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(5, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, requestMessages.Count);
foreach (var requestMessage in requestMessages)
{
Assert.AreEqual("World", requestMessage.Name);
}
}
public async Task Unary_MultipleLargeMessages_ExceedChannelMaxBufferSize()
{
// Arrange
var sp1 = new SyncPoint(runContinuationsAsynchronously: true);
var sp2 = new SyncPoint(runContinuationsAsynchronously: true);
var sp3 = new SyncPoint(runContinuationsAsynchronously: true);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig(),
maxRetryBufferSize: 200,
maxRetryBufferPerCallSize: 100);
var request = new DataMessage {
Data = ByteString.CopyFrom(new byte[90])
};
// Act
var call1Task = MakeCall(Fixture, channel, request, sp1);
await sp1.WaitForSyncPoint();
var call2Task = MakeCall(Fixture, channel, request, sp2);
await sp2.WaitForSyncPoint();
// Will exceed channel buffer limit and won't retry
var call3Task = MakeCall(Fixture, channel, request, sp3);
await sp3.WaitForSyncPoint();
// Assert
Assert.AreEqual(194, channel.CurrentRetryBufferSize);
sp1.Continue();
sp2.Continue();
sp3.Continue();
var response = await call1Task.DefaultTimeout();
Assert.AreEqual(90, response.Data.Length);
response = await call2Task.DefaultTimeout();
Assert.AreEqual(90, response.Data.Length);
// Can't retry because buffer size exceeded.
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call3Task).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
public async Task ServerStreaming_CancellatonTokenSpecified_TokenUnregisteredAndResourcesReleased()
{
Task FakeServerStreamCall(DataMessage request, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context)
{
return(Task.CompletedTask);
}
// Arrange
var method = Fixture.DynamicGrpc.AddServerStreamingMethod <DataMessage, DataMessage>(FakeServerStreamCall);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var references = new List <WeakReference>();
// Checking that token register calls don't build up on CTS and create a memory leak.
var cts = new CancellationTokenSource();
// Act
// Send calls in a different method so there is no chance that a stack reference
// to a gRPC call is still alive after calls are complete.
await MakeCallsAsync(channel, method, references, cts.Token).DefaultTimeout();
// Assert
// There is a race when cleaning up cancellation token registry.
// Retry a few times to ensure GC is run after unregister.
await TestHelpers.AssertIsTrueRetryAsync(() =>
{
GC.Collect();
GC.WaitForPendingFinalizers();
for (var i = 0; i < references.Count; i++)
{
if (references[i].IsAlive)
{
return(false);
}
}
// Resources for past calls were successfully GCed.
return(true);
}, "Assert that retry call resources are released.");
}
public async Task Duplex_DeadlineExceedDuringDelay_Failure()
{
var callCount = 0;
Task DuplexDeadlineExceeded(IAsyncStreamReader <DataMessage> requestStream, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context)
{
callCount++;
return(Task.FromException(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata
{
new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <DataMessage, DataMessage>(DuplexDeadlineExceeded);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var deadlineTimeout = 500;
var call = client.DuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(deadlineTimeout)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseStream.MoveNext(CancellationToken.None)).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.RequestStream.WriteAsync(new DataMessage())).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
}
public async Task Unary_DeadlineExceedAfterServerCall_Failure(int exceptedServerCallCount)
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
if (callCount < exceptedServerCallCount)
{
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""))));
}
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig(retryableStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(200)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(exceptedServerCallCount, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
tcs.SetResult(new DataMessage());
}
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async(request, ct) =>
{
// Ensure SendAsync call doesn't hang upon cancellation by gRPC client.
using var registration = ct.Register(() => tcs.TrySetCanceled());
Interlocked.Increment(ref callCount);
return(await tcs.Task);
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: DeadlineExceeded", write.State.ToString());
}
public async Task ClientStreamWriter_WriteWhileComplete_ErrorThrown()
{
// Arrange
var streamContent = new SyncPointMemoryStream();
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
return(Task.FromResult(ResponseUtils.CreateResponse(HttpStatusCode.OK, new StreamContent(streamContent))));
});
var serviceConfig = ServiceConfigHelpers.CreateRetryServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
await call.RequestStream.CompleteAsync().DefaultTimeout();
var resultTask = call.ResponseAsync;
// Assert
var writeException1 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", writeException1.Message);
await streamContent.AddDataAndWait(await ClientTestHelpers.GetResponseDataAsync(new HelloReply
{
Message = "Hello world 1"
}).DefaultTimeout()).DefaultTimeout();
await streamContent.AddDataAndWait(new byte[0]);
var result = await resultTask.DefaultTimeout();
Assert.AreEqual("Hello world 1", result.Message);
var writeException2 = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "2"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", writeException2.Message);
}
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateRetryServiceConfig());
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage
{
Data = ByteString.CopyFrom(new byte[payloadSize])
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
if (!exceedBufferLimit)
{
Assert.AreEqual(5, callCount);
}
else
{
Assert.AreEqual(1, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded");
}
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
}
public async Task Unary_DeadlineExceedDuringDelay_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata
{
new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(300)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
}
