public async Task ResetDeadline()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset does not change the absolute deadline of the call.
// The next retry attempt will therefore fail.
state.Reset();
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task ConsecutiveErrors_FailsRetryWhenDeadlineExceeded()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 6 seconds.
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task ErrorWithBackoffAfterDeadline_FailsRetry()
{
// Create a clock that starts at zero ticks.
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(10).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
// The retry info contains a wait time that is past the deadline of the call.
// The retry state will throw a DeadlineExceeded error without waiting.
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(20))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Check that the clock has not been advanced to verify that the retry state did not wait 20 seconds
// before throwing an exception.
Assert.Equal(0, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RecordErrorAndWait_RetryInfo()
{
var mock = new Mock <IScheduler>(MockBehavior.Strict);
// Delay taken from retry info
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(3), default)).Returns(Task.FromResult(0));
// Delay taken from backoff settings (which weren't affected by the first exception, because it contained backoff information)
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
// The first exception contains retry info, so we don't use the backoff settings
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception1 = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
var exception2 = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
RetryState state = new RetryState(new FakeClock(), mock.Object, s_retrySettings, s_callSettings);
Assert.True(state.CanRetry(exception1));
await state.WaitAsync(exception1, default);
Assert.True(state.CanRetry(exception2));
await state.WaitAsync(exception2, default);
}
public async Task RecordErrorAndWait_RetryInfo()
{
var settings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
// Delay taken from retry info
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(3), default)).Returns(Task.FromResult(0));
// Delay taken from backoff settings (which have still doubled, even when the first value wasn't used)
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
// The first exception contains retry info, so we don't use the backoff settings
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception1 = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
var exception2 = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
RetryState state = new RetryState(mock.Object, settings, RetrySettings.NoJitter, maxConsecutiveErrors: 5);
Assert.True(state.CanRetry(exception1));
await state.RecordErrorAndWaitAsync(exception1, default);
Assert.True(state.CanRetry(exception2));
await state.RecordErrorAndWaitAsync(exception2, default);
}
public void CanRetry_ResourceExhausted_WithRetryInfo()
{
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(2))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.ResourceExhausted, "Bang"), trailers);
RetryState state = CreateSimpleRetryState();
Assert.True(state.CanRetry(exception));
}
public async Task ResetTimeout()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(7)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset should set the deadline of the call to CurrentTime + Timeout.
// That means that we can do two new retries without a timeout exception.
state.Reset();
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 12 seconds.
Assert.Equal(TimeSpan.FromSeconds(12).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}