public async Task Unary_ExceedAttempts_Failure(int?hedgingDelay)
{
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Ignore errors
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var delay = (hedgingDelay == null)
? (TimeSpan?)null
: TimeSpan.FromMilliseconds(hedgingDelay.Value);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: delay));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task Unary_DeadlineExceedBeforeServerCall_Failure()
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(0, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
tcs.SetResult(new DataMessage());
}
public async Task Unary_TriggerRetryThrottling_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
retryThrottling: new RetryThrottlingPolicy
{
MaxTokens = 5,
TokenRatio = 0.1
}));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(3, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: Throttled");
}
public async Task Unary_AttemptsGreaterThanDefaultClientLimit_LimitedAttemptsMade(int hedgingDelay)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 10, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelay)));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, callCount);
AssertHasLog(LogLevel.Debug, "MaxAttemptsLimited", "The method has 10 attempts specified in the service config. The number of attempts has been limited by channel configuration to 5.");
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ExceededAttemptCount");
}
public async Task AsyncUnaryCall_FatalStatusCode_HedgeDelay_Failure()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, (callCount == 1) ? StatusCode.Unavailable : StatusCode.InvalidArgument));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(50));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.InvalidArgument, ex.StatusCode);
Assert.AreEqual(StatusCode.InvalidArgument, call.GetStatus().StatusCode);
Assert.AreEqual(2, callCount);
}
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure()
{
// Arrange
var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(request =>
{
Interlocked.Increment(ref callCount);
return(tcs.Task);
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
}
public async Task Unary_LargeMessages_ExceedPerCallBufferSize(long payloadSize, bool exceedBufferLimit, int hedgingDelayMilliseconds)
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.Unavailable, ""))));
}
// Ignore errors
SetExpectedErrorsFilter(writeContext =>
{
if (writeContext.EventId.Name == "ErrorSendingMessage" ||
writeContext.EventId.Name == "ErrorExecutingServiceMethod")
{
return(true);
}
return(false);
});
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMilliseconds)),
maxReceiveMessageSize: (int)GrpcChannel.DefaultMaxRetryBufferPerCallSize * 2);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage
{
Data = ByteString.CopyFrom(new byte[payloadSize])
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
if (!exceedBufferLimit)
{
Assert.AreEqual(5, callCount);
}
else
{
Assert.AreEqual(1, callCount);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: BufferExceeded");
// Cancelled calls could cause server errors. Delay so these error don't show up
// in the next unit test.
await Task.Delay(100);
}
Assert.AreEqual(0, channel.CurrentRetryBufferSize);
}
public async Task ClientStreaming_WriteAsyncCancellationDuringRetry_Canceled(bool throwOperationCanceledOnCancellation)
{
async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
Logger.LogInformation("Server reading message 1.");
Assert.IsTrue(await requestStream.MoveNext());
Logger.LogInformation("Server pausing.");
await Task.Delay(TimeSpan.FromMilliseconds(500));
Logger.LogInformation("Server erroring.");
throw new RpcException(new Status(StatusCode.Unavailable, string.Empty));
}
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromSeconds(20)),
maxReceiveMessageSize: BigMessageSize * 2,
maxRetryBufferPerCallSize: BigMessageSize * 2,
throwOperationCanceledOnCancellation: throwOperationCanceledOnCancellation);
var client = TestClientFactory.Create(channel, method);
// Act
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var call = client.ClientStreamingCall();
Logger.LogInformation("Client writing message 1.");
await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }, cts.Token).DefaultTimeout();
Logger.LogInformation("Client writing message 2.");
var writeTask = call.RequestStream.WriteAsync(new DataMessage {
Data = ByteString.CopyFrom(new byte[BigMessageSize])
}, cts.Token);
// Assert
if (throwOperationCanceledOnCancellation)
{
var ex = await ExceptionAssert.ThrowsAsync <OperationCanceledException>(() => writeTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, call.GetStatus().StatusCode);
}
else
{
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => writeTask).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, ex.StatusCode);
}
Assert.IsTrue(cts.Token.IsCancellationRequested, "WriteAsync finished when CancellationToken wasn't triggered.");
}
public async Task AsyncServerStreamingCall_SuccessAfterRetry_RequestContentSent()
{
// Arrange
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
MemoryStream?requestContent = null;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
var s = await request.Content !.ReadAsStreamAsync();
var ms = new MemoryStream();
await s.CopyToAsync(ms);
if (callCount == 1)
{
await syncPoint.WaitForSyncPoint();
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
syncPoint.Continue();
requestContent = ms;
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(50));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncServerStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ServerStreaming), string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
var moveNextTask = call.ResponseStream.MoveNext(CancellationToken.None);
// Wait until the first call has failed and the second is on the server
await syncPoint.WaitToContinue().DefaultTimeout();
// Assert
Assert.IsTrue(await moveNextTask);
Assert.AreEqual("Hello world", call.ResponseStream.Current.Message);
requestContent !.Seek(0, SeekOrigin.Begin);
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
Assert.AreEqual("World", requestMessage !.Name);
}
public async Task Unary_RetryThrottlingBecomesActive_HasDelay_Failure()
{
var callCount = 0;
var syncPoint = new SyncPoint(runContinuationsAsynchronously: true);
async Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
Interlocked.Increment(ref callCount);
await syncPoint.WaitToContinue();
return(request);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromMilliseconds(100),
retryThrottling: new RetryThrottlingPolicy
{
MaxTokens = 5,
TokenRatio = 0.1
}));
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage());
await syncPoint.WaitForSyncPoint().DefaultTimeout();
// Manually trigger retry throttling
Debug.Assert(channel.RetryThrottling != null);
channel.RetryThrottling.CallFailure();
channel.RetryThrottling.CallFailure();
channel.RetryThrottling.CallFailure();
Debug.Assert(channel.RetryThrottling.IsRetryThrottlingActive());
// Assert
await TestHelpers.AssertIsTrueRetryAsync(() => HasLog(LogLevel.Debug, "AdditionalCallsBlockedByRetryThrottling", "Additional calls blocked by retry throttling."), "Check for expected log.");
Assert.AreEqual(1, callCount);
syncPoint.Continue();
await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: ResponseHeadersReceived");
}
public async Task Dispose_ActiveCalls_CleansUpActiveCalls()
{
// Arrange
var allCallsOnServerTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var waitUntilFinishedTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
// All calls are in-progress at once.
Interlocked.Increment(ref callCount);
if (callCount == 5)
{
allCallsOnServerTcs.SetResult(null);
}
await waitUntilFinishedTcs.Task;
await request.Content !.CopyToAsync(new MemoryStream());
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMilliseconds(20));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
var hedgingCall = new HedgingCall <HelloRequest, HelloReply>(CreateHedgingPolicy(serviceConfig.MethodConfigs[0].HedgingPolicy), invoker.Channel, ClientTestHelpers.ServiceMethod, new CallOptions());
// Act
hedgingCall.StartUnary(new HelloRequest {
Name = "World"
});
Assert.IsFalse(hedgingCall.CreateHedgingCallsTask !.IsCompleted);
// Assert
Assert.AreEqual(1, hedgingCall._activeCalls.Count);
await allCallsOnServerTcs.Task.DefaultTimeout();
Assert.AreEqual(5, callCount);
Assert.AreEqual(5, hedgingCall._activeCalls.Count);
hedgingCall.Dispose();
Assert.AreEqual(0, hedgingCall._activeCalls.Count);
await hedgingCall.CreateHedgingCallsTask !.DefaultTimeout();
waitUntilFinishedTcs.SetResult(null);
}
public async Task AsyncUnaryCall_ExceedAttempts_Failure()
{
// Arrange
var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var requestMessages = new List <HelloRequest>();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
// All calls are in-progress at once.
Interlocked.Increment(ref callCount);
if (callCount == 5)
{
tcs.TrySetResult(null);
}
await tcs.Task;
var requestContent = await request.Content !.ReadAsStreamAsync();
var requestMessage = await ReadRequestMessage(requestContent);
lock (requestMessages)
{
requestMessages.Add(requestMessage !);
}
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(5, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, requestMessages.Count);
foreach (var requestMessage in requestMessages)
{
Assert.AreEqual("World", requestMessage.Name);
}
}
public async Task Duplex_DeadlineExceedDuringDelay_Failure()
{
var callCount = 0;
Task DuplexDeadlineExceeded(IAsyncStreamReader <DataMessage> requestStream, IServerStreamWriter <DataMessage> responseStream, ServerCallContext context)
{
callCount++;
return(Task.FromException(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata
{
new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <DataMessage, DataMessage>(DuplexDeadlineExceeded);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var deadlineTimeout = 500;
var call = client.DuplexStreamingCall(new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(deadlineTimeout)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseStream.MoveNext(CancellationToken.None)).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.RequestStream.WriteAsync(new DataMessage())).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
}
public async Task AsyncUnaryCall_ExceedDeadlineWithActiveCalls_Failure()
{
// Arrange
var testSink = new TestSink();
var services = new ServiceCollection();
services.AddLogging(b =>
{
b.AddProvider(new TestLoggerProvider(testSink));
});
services.AddNUnitLogger();
var provider = services.BuildServiceProvider();
var tcs = new TaskCompletionSource <HttpResponseMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async(request, ct) =>
{
// Ensure SendAsync call doesn't hang upon cancellation by gRPC client.
using var registration = ct.Register(() => tcs.TrySetCanceled());
Interlocked.Increment(ref callCount);
return(await tcs.Task);
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(hedgingDelay: TimeSpan.FromMilliseconds(200));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, loggerFactory: provider.GetRequiredService <ILoggerFactory>(), serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(100)), new HelloRequest {
Name = "World"
});
// Assert
Assert.AreEqual(1, callCount);
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
var write = testSink.Writes.Single(w => w.EventId.Name == "CallCommited");
Assert.AreEqual("Call commited. Reason: DeadlineExceeded", write.State.ToString());
}
public async Task AsyncUnaryCall_OneAttempt_Success(int maxAttempts)
{
// Arrange
var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
await tcs.Task;
await request.Content !.CopyToAsync(new MemoryStream());
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: maxAttempts);
var invoker = HttpClientCallInvokerFactory.Create(
httpClient,
serviceConfig: serviceConfig,
configure: o => o.MaxRetryAttempts = maxAttempts);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
await TestHelpers.AssertIsTrueRetryAsync(() => callCount == maxAttempts, "All calls made at once.");
tcs.SetResult(null);
var rs = await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual("Hello world", rs.Message);
Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
}
public async Task AsyncClientStreamingCall_CompleteAndWriteAfterResult_Error()
{
// Arrange
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
_ = request.Content !.ReadAsStreamAsync();
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
var responseMessage = await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual("Hello world", responseMessage.Message);
requestContent.Seek(0, SeekOrigin.Begin);
await call.RequestStream.CompleteAsync().DefaultTimeout();
var ex = await ExceptionAssert.ThrowsAsync <InvalidOperationException>(() => call.RequestStream.WriteAsync(new HelloRequest {
Name = "1"
})).DefaultTimeout();
Assert.AreEqual("Request stream has already been completed.", ex.Message);
}
public async Task AsyncUnaryCall_PushbackDelay_PushbackDelayUpdatesNextCallDelay()
{
// Arrange
var stopwatch = new Stopwatch();
var callIntervals = new List <long>();
var hedgingDelay = TimeSpan.FromSeconds(10);
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callIntervals.Add(stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
Interlocked.Increment(ref callCount);
await request.Content !.CopyToAsync(new MemoryStream());
var hedgingPushback = hedgingDelay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
if (callCount == 1)
{
hedgingPushback = "0";
}
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: hedgingPushback));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: hedgingDelay);
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
stopwatch.Start();
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
await TestHelpers.AssertIsTrueRetryAsync(() => callIntervals.Count == 2, "Only two calls should be made.").DefaultTimeout();
// First call should happen immediately
Assert.LessOrEqual(callIntervals[0], 100);
// Second call should happen after delay
Assert.LessOrEqual(callIntervals[1], hedgingDelay.TotalMilliseconds);
}
public async Task Unary_DeadlineExceedDuringDelay_Failure()
{
var callCount = 0;
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""), new Metadata
{
new Metadata.Entry(GrpcProtocolConstants.RetryPushbackHeader, TimeSpan.FromSeconds(10).TotalMilliseconds.ToString())
})));
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
hedgingDelay: TimeSpan.FromSeconds(10),
nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(300)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.AreEqual(1, callCount);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
AssertHasLog(LogLevel.Debug, "CallCommited", "Call commited. Reason: DeadlineExceeded");
}
public async Task AsyncUnaryCall_ExceedAttempts_PusbackDelay_Failure()
{
// Arrange
var stopwatch = new Stopwatch();
var callIntervals = new List <long>();
var hedgeDelay = TimeSpan.FromMilliseconds(100);
const int timerResolutionMs = 15 * 2; // Timer has a precision of about 15ms. Double it, just to be safe
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
callIntervals.Add(stopwatch.ElapsedMilliseconds);
stopwatch.Restart();
Interlocked.Increment(ref callCount);
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable, retryPushbackHeader: hedgeDelay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture)));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: hedgeDelay);
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
stopwatch.Start();
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(2, callCount);
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
// First call should happen immediately
Assert.LessOrEqual(callIntervals[0], hedgeDelay.TotalMilliseconds);
// Second call should happen after delay
Assert.GreaterOrEqual(callIntervals[1], hedgeDelay.TotalMilliseconds - timerResolutionMs);
}
public async Task Unary_DeadlineExceedAfterServerCall_Failure(int exceptedServerCallCount)
{
var callCount = 0;
var tcs = new TaskCompletionSource <DataMessage>(TaskCreationOptions.RunContinuationsAsynchronously);
Task <DataMessage> UnaryFailure(DataMessage request, ServerCallContext context)
{
callCount++;
if (callCount < exceptedServerCallCount)
{
return(Task.FromException <DataMessage>(new RpcException(new Status(StatusCode.DeadlineExceeded, ""))));
}
return(tcs.Task);
}
// Arrange
var method = Fixture.DynamicGrpc.AddUnaryMethod <DataMessage, DataMessage>(UnaryFailure);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(nonFatalStatusCodes: new List <StatusCode> {
StatusCode.DeadlineExceeded
});
var channel = CreateChannel(serviceConfig: serviceConfig);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.UnaryCall(new DataMessage(), new CallOptions(deadline: DateTime.UtcNow.AddMilliseconds(200)));
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.DeadlineExceeded, ex.StatusCode);
Assert.AreEqual(StatusCode.DeadlineExceeded, call.GetStatus().StatusCode);
Assert.IsFalse(Logs.Any(l => l.EventId.Name == "DeadlineTimerRescheduled"));
}
public async Task AsyncUnaryCall_ManyAttemptsNoDelay_MarshallerCalledOnce()
{
// Arrange
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
Interlocked.Increment(ref callCount);
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig();
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
var marshallerCount = 0;
var requestMarshaller = Marshallers.Create <HelloRequest>(
r =>
{
Interlocked.Increment(ref marshallerCount);
return(r.ToByteArray());
},
data => HelloRequest.Parser.ParseFrom(data));
var method = ClientTestHelpers.GetServiceMethod(requestMarshaller: requestMarshaller);
// Act
var call = invoker.AsyncUnaryCall <HelloRequest, HelloReply>(method, string.Empty, new CallOptions(), new HelloRequest {
Name = "World"
});
// Assert
var ex = await ExceptionAssert.ThrowsAsync <RpcException>(() => call.ResponseAsync).DefaultTimeout();
Assert.AreEqual(StatusCode.Unavailable, ex.StatusCode);
Assert.AreEqual(StatusCode.Unavailable, call.GetStatus().StatusCode);
Assert.AreEqual(5, callCount);
Assert.AreEqual(1, marshallerCount);
}
public async Task Duplex_ManyParallelRequests_MessageRoundTripped()
{
const string ImportantMessage =
@" _____ _____ _____
| __ \| __ \ / ____|
__ _| |__) | |__) | |
/ _` | _ /| ___/| |
| (_| | | \ \| | | |____
\__, |_| \_\_| \_____|
__/ |
|___/
_
(_)
_ ___
| / __|
| \__ \ _
|_|___/ | |
___ ___ ___ | |
/ __/ _ \ / _ \| |
| (_| (_) | (_) | |
\___\___/ \___/|_|
";
var attempts = 100;
var allUploads = new List <string>();
var allCompletedTasks = new List <Task>();
var tcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
async Task MessageUpload(
IAsyncStreamReader <StringValue> requestStream,
IServerStreamWriter <StringValue> responseStream,
ServerCallContext context)
{
// Receive chunks
var chunks = new List <string>();
await foreach (var chunk in requestStream.ReadAllAsync())
{
chunks.Add(chunk.Value);
}
Task completeTask;
lock (allUploads)
{
allUploads.Add(string.Join(Environment.NewLine, chunks));
if (allUploads.Count < attempts)
{
// Check that unused calls are canceled.
completeTask = Task.Run(async() =>
{
await tcs.Task;
var cancellationTcs = new TaskCompletionSource <bool>();
context.CancellationToken.Register(s => ((TaskCompletionSource <bool>)s !).SetResult(true), cancellationTcs);
await cancellationTcs.Task;
});
}
else
{
// Write response in used call.
completeTask = Task.Run(async() =>
{
// Write chunks
foreach (var chunk in chunks)
{
await responseStream.WriteAsync(new StringValue
{
Value = chunk
});
}
});
}
}
await completeTask;
}
var method = Fixture.DynamicGrpc.AddDuplexStreamingMethod <StringValue, StringValue>(MessageUpload);
var channel = CreateChannel(serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 100, hedgingDelay: TimeSpan.Zero), maxRetryAttempts: 100);
var client = TestClientFactory.Create(channel, method);
using var call = client.DuplexStreamingCall();
var lines = ImportantMessage.Split(Environment.NewLine);
for (var i = 0; i < lines.Length; i++)
{
await call.RequestStream.WriteAsync(new StringValue { Value = lines[i] }).DefaultTimeout();
await Task.Delay(TimeSpan.FromSeconds(0.01)).DefaultTimeout();
}
await call.RequestStream.CompleteAsync().DefaultTimeout();
await TestHelpers.AssertIsTrueRetryAsync(() => allUploads.Count == 100, "Wait for all calls to reach server.").DefaultTimeout();
tcs.SetResult(null);
var receivedLines = new List <string>();
await foreach (var line in call.ResponseStream.ReadAllAsync().DefaultTimeout())
{
receivedLines.Add(line.Value);
}
Assert.AreEqual(ImportantMessage, string.Join(Environment.NewLine, receivedLines));
foreach (var upload in allUploads)
{
Assert.AreEqual(ImportantMessage, upload);
}
await Task.WhenAll(allCompletedTasks).DefaultTimeout();
}
public async Task ClientStreaming_WriteLargeMessageCausingCommit_Success()
{
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
var firstMessageTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var clientCancellationTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var serverCanceledTcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
const int maxAttempts = 2;
var callCount = 0;
var serverSuccessCount = 0;
var serverAbortCount = 0;
async Task <DataMessage> ClientStreamingWithCancellation(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
Logger.LogInformation("Server reading first message.");
Assert.IsTrue(await requestStream.MoveNext());
if (Interlocked.Increment(ref callCount) >= maxAttempts)
{
firstMessageTcs.TrySetResult(null);
}
try
{
await requestStream.MoveNext();
Interlocked.Increment(ref serverSuccessCount);
return(requestStream.Current);
}
catch (Exception ex)
{
Interlocked.Increment(ref serverAbortCount);
serverCanceledTcs.TrySetException(ex);
throw;
}
}
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithCancellation);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
maxAttempts: maxAttempts);
var channel = CreateChannel(serviceConfig: serviceConfig, maxReceiveMessageSize: BigMessageSize * 2);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.ClientStreamingCall();
// Assert
Logger.LogInformation("Client sending first message.");
await call.RequestStream.WriteAsync(
new DataMessage { Data = ByteString.CopyFrom(Encoding.UTF8.GetBytes("Hello world")) },
CancellationToken.None).DefaultTimeout();
await firstMessageTcs.Task.DefaultTimeout();
// This large message causes the call to be commited and cancels other calls.
Logger.LogInformation("Client sending large message.");
var writeLargeMessageTask = call.RequestStream.WriteAsync(
new DataMessage {
Data = ByteString.CopyFrom(new byte[BigMessageSize])
});
var response = await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual(BigMessageSize, response.Data.Length);
await ExceptionAssert.ThrowsAsync <Exception>(() => serverCanceledTcs.Task).DefaultTimeout();
Assert.AreEqual(1, serverSuccessCount);
Assert.AreEqual(1, serverAbortCount);
}
public async Task ClientStreaming_WriteAsyncCancellationDuring_ClientAbort()
{
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
var firstMessageTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var clientCancellationTcs = new TaskCompletionSource <object?>(TaskCreationOptions.RunContinuationsAsynchronously);
var serverCanceledTcs = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var maxAttempts = 2;
var callCount = 0;
async Task <DataMessage> ClientStreamingWithCancellation(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
Logger.LogInformation("Server reading first message.");
Assert.IsTrue(await requestStream.MoveNext());
if (Interlocked.Increment(ref callCount) >= maxAttempts)
{
firstMessageTcs.TrySetResult(null);
}
Logger.LogInformation("Server waiting for canceled client message.");
await clientCancellationTcs.Task;
try
{
await requestStream.MoveNext();
throw new Exception("Should never reached here.");
}
catch (Exception ex)
{
if (IsWriteCanceledException(ex))
{
serverCanceledTcs.SetResult(context.CancellationToken.IsCancellationRequested);
return(new DataMessage());
}
serverCanceledTcs.TrySetException(ex);
throw;
}
}
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithCancellation);
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(
maxAttempts: maxAttempts);
var channel = CreateChannel(
serviceConfig: serviceConfig,
maxReceiveMessageSize: BigMessageSize * 2);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.ClientStreamingCall();
// Assert
Logger.LogInformation("Client sending first message.");
await call.RequestStream.WriteAsync(
new DataMessage { Data = ByteString.CopyFrom(Encoding.UTF8.GetBytes("Hello world")) },
CancellationToken.None).DefaultTimeout();
await firstMessageTcs.Task.DefaultTimeout();
Logger.LogInformation("Client sending large message.");
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var clientEx = await ExceptionAssert.ThrowsAsync <RpcException>(
() => call.RequestStream.WriteAsync(
new DataMessage {
Data = ByteString.CopyFrom(new byte[BigMessageSize])
},
cts.Token)).DefaultTimeout();
Assert.AreEqual(StatusCode.Cancelled, clientEx.StatusCode);
clientCancellationTcs.SetResult(null);
Assert.IsTrue(await serverCanceledTcs.Task.DefaultTimeout());
}
public async Task ClientStreaming_WriteAsyncFailsUntilRetries_WriteAsyncAwaitsUntilSuccess()
{
Task?largeWriteTask = null;
var callCount = 0;
bool?clientWriteWaitedForServerRead = null;
async Task <DataMessage> ClientStreamingWithReadFailures(IAsyncStreamReader <DataMessage> requestStream, ServerCallContext context)
{
Logger.LogInformation("Server reading message 1.");
Assert.IsTrue(await requestStream.MoveNext());
var currentCallCount = Interlocked.Increment(ref callCount);
Logger.LogInformation("Server current call count: " + currentCallCount);
if (currentCallCount <= 2)
{
Logger.LogInformation("Server pausing.");
await Task.Delay(TimeSpan.FromMilliseconds(500));
Logger.LogInformation("Server erroring.");
throw new RpcException(new Status(StatusCode.Unavailable, string.Empty));
}
else
{
clientWriteWaitedForServerRead = !largeWriteTask !.IsCompleted;
Logger.LogInformation("Server reading message 2.");
Assert.IsTrue(await requestStream.MoveNext());
Logger.LogInformation("Server sending response.");
return(new DataMessage());
}
}
SetExpectedErrorsFilter(writeContext =>
{
return(true);
});
// Arrange
var method = Fixture.DynamicGrpc.AddClientStreamingMethod <DataMessage, DataMessage>(ClientStreamingWithReadFailures);
var channel = CreateChannel(
serviceConfig: ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 5, hedgingDelay: TimeSpan.FromMinutes(20)),
maxReceiveMessageSize: BigMessageSize * 2,
maxRetryBufferPerCallSize: BigMessageSize * 2);
var client = TestClientFactory.Create(channel, method);
// Act
var call = client.ClientStreamingCall();
Logger.LogInformation("Client writing message 1.");
await call.RequestStream.WriteAsync(new DataMessage { Data = ByteString.CopyFrom(new byte[] { (byte)1 }) }).DefaultTimeout();
Logger.LogInformation("Client writing message 2.");
largeWriteTask = call.RequestStream.WriteAsync(new DataMessage {
Data = ByteString.CopyFrom(new byte[BigMessageSize])
});
await largeWriteTask.DefaultTimeout();
// Assert
Logger.LogInformation("Client waiting for response.");
var response = await call.ResponseAsync.DefaultTimeout();
Assert.IsTrue(clientWriteWaitedForServerRead);
}
public async Task AsyncClientStreamingCall_SuccessAfterRetry_RequestContentSent(int hedgingDelayMS)
{
// Arrange
var callLock = new object();
var requestContent = new MemoryStream();
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
var firstCall = false;
lock (callLock)
{
callCount++;
if (callCount == 1)
{
firstCall = true;
}
}
if (firstCall)
{
await request.Content !.CopyToAsync(new MemoryStream());
return(ResponseUtils.CreateHeadersOnlyResponse(HttpStatusCode.OK, StatusCode.Unavailable));
}
var content = (PushStreamContent <HelloRequest, HelloReply>)request.Content !;
await content.PushComplete.DefaultTimeout();
await request.Content !.CopyToAsync(requestContent);
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: 2, hedgingDelay: TimeSpan.FromMilliseconds(hedgingDelayMS));
var invoker = HttpClientCallInvokerFactory.Create(httpClient, serviceConfig: serviceConfig);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.GetServiceMethod(MethodType.ClientStreaming), string.Empty, new CallOptions());
// Assert
Assert.IsNotNull(call);
var responseTask = call.ResponseAsync;
Assert.IsFalse(responseTask.IsCompleted, "Response not returned until client stream is complete.");
await call.RequestStream.WriteAsync(new HelloRequest { Name = "1" }).DefaultTimeout();
await call.RequestStream.WriteAsync(new HelloRequest { Name = "2" }).DefaultTimeout();
await call.RequestStream.CompleteAsync().DefaultTimeout();
var responseMessage = await responseTask.DefaultTimeout();
Assert.AreEqual("Hello world", responseMessage.Message);
requestContent.Seek(0, SeekOrigin.Begin);
var requests = new List <HelloRequest>();
while (true)
{
var requestMessage = await ReadRequestMessage(requestContent).DefaultTimeout();
if (requestMessage == null)
{
break;
}
requests.Add(requestMessage);
}
Assert.AreEqual(2, requests.Count);
Assert.AreEqual("1", requests[0].Name);
Assert.AreEqual("2", requests[1].Name);
}
public async Task AsyncClientStreamingCall_ManyParallelCalls_ReadDirectlyToRequestStream()
{
// Arrange
var requestStreams = new List <WriterTestStream>();
var attempts = 100;
var callCount = 0;
var httpClient = ClientTestHelpers.CreateTestClient(async request =>
{
WriterTestStream writerTestStream;
lock (requestStreams)
{
Interlocked.Increment(ref callCount);
writerTestStream = new WriterTestStream();
requestStreams.Add(writerTestStream);
}
await request.Content !.CopyToAsync(writerTestStream);
var reply = new HelloReply {
Message = "Hello world"
};
var streamContent = await ClientTestHelpers.CreateResponseContent(reply).DefaultTimeout();
return(ResponseUtils.CreateResponse(HttpStatusCode.OK, streamContent));
});
var serviceConfig = ServiceConfigHelpers.CreateHedgingServiceConfig(maxAttempts: attempts);
var invoker = HttpClientCallInvokerFactory.Create(
httpClient,
serviceConfig: serviceConfig,
configure: o => o.MaxRetryAttempts = attempts);
// Act
var call = invoker.AsyncClientStreamingCall <HelloRequest, HelloReply>(ClientTestHelpers.ServiceMethod, string.Empty, new CallOptions());
var writeAsyncTask = call.RequestStream.WriteAsync(new HelloRequest {
Name = "World"
});
// Assert
await TestHelpers.AssertIsTrueRetryAsync(() => callCount == attempts, "All calls made at once.");
var firstMessages = await Task.WhenAll(requestStreams.Select(s => s.WaitForDataAsync())).DefaultTimeout();
await writeAsyncTask.DefaultTimeout();
foreach (var message in firstMessages)
{
Assert.IsTrue(firstMessages[0].Span.SequenceEqual(message.Span));
}
writeAsyncTask = call.RequestStream.WriteAsync(new HelloRequest {
Name = "World 2"
});
var secondMessages = await Task.WhenAll(requestStreams.Select(s => s.WaitForDataAsync())).DefaultTimeout();
await writeAsyncTask.DefaultTimeout();
foreach (var message in secondMessages)
{
Assert.IsTrue(secondMessages[0].Span.SequenceEqual(message.Span));
}
await call.RequestStream.CompleteAsync().DefaultTimeout();
var rs = await call.ResponseAsync.DefaultTimeout();
Assert.AreEqual("Hello world", rs.Message);
Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode);
}
