public async Task ResetDeadline()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset does not change the absolute deadline of the call.
// The next retry attempt will therefore fail.
state.Reset();
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task ConsecutiveErrors_FailsRetryWhenDeadlineExceeded()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(7).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 6 seconds.
Assert.Equal(TimeSpan.FromSeconds(6).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task RecordErrorAndWait_RetryInfo()
{
var mock = new Mock <IScheduler>(MockBehavior.Strict);
// Delay taken from retry info
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(3), default)).Returns(Task.FromResult(0));
// Delay taken from backoff settings (which weren't affected by the first exception, because it contained backoff information)
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
// The first exception contains retry info, so we don't use the backoff settings
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception1 = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
var exception2 = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
RetryState state = new RetryState(new FakeClock(), mock.Object, s_retrySettings, s_callSettings);
Assert.True(state.CanRetry(exception1));
await state.WaitAsync(exception1, default);
Assert.True(state.CanRetry(exception2));
await state.WaitAsync(exception2, default);
}
public async Task RecordErrorAndWait_RetryInfo()
{
var settings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
// Delay taken from retry info
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(3), default)).Returns(Task.FromResult(0));
// Delay taken from backoff settings (which have still doubled, even when the first value wasn't used)
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
// The first exception contains retry info, so we don't use the backoff settings
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception1 = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
var exception2 = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
RetryState state = new RetryState(mock.Object, settings, RetrySettings.NoJitter, maxConsecutiveErrors: 5);
Assert.True(state.CanRetry(exception1));
await state.RecordErrorAndWaitAsync(exception1, default);
Assert.True(state.CanRetry(exception2));
await state.RecordErrorAndWaitAsync(exception2, default);
}
public void CanRetry_SimpleStatusCodes(StatusCode code, bool expectedRetriable)
{
var exception = new RpcException(new Status(code, "Bang"));
RetryState state = CreateSimpleRetryState();
Assert.Equal(expectedRetriable, state.CanRetry(exception));
}
public async Task ErrorWithBackoffAfterDeadline_FailsRetry()
{
// Create a clock that starts at zero ticks.
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromDeadline(new DateTime(TimeSpan.FromSeconds(10).Ticks, DateTimeKind.Utc)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
// The retry info contains a wait time that is past the deadline of the call.
// The retry state will throw a DeadlineExceeded error without waiting.
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(20))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Check that the clock has not been advanced to verify that the retry state did not wait 20 seconds
// before throwing an exception.
Assert.Equal(0, clock.GetCurrentDateTimeUtc().Ticks);
}
public async Task CanRetry_MaxConsecutiveRetries_NoReset()
{
var state = new RetryState(
new NoOpScheduler(),
s_retrySettings,
maxConsecutiveErrors: 2);
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.False(state.CanRetry(exception));
}
public async Task CanRetry_NoReset()
{
var state = new RetryState(
new FakeClock(),
new NoOpScheduler(),
s_retrySettings,
s_callSettings);
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
}
public async Task CanRetry_MaxConsecutiveRetries_NoReset()
{
var state = new RetryState(
new NoOpScheduler(),
new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(15), 2.0),
RetrySettings.NoJitter,
maxConsecutiveErrors: 2);
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.False(state.CanRetry(exception));
}
public void CanRetry_ResourceExhausted_NoRetryInfo()
{
Metadata trailers = new Metadata
{
{ "otherinfo", "value" }
};
var exception = new RpcException(new Status(StatusCode.ResourceExhausted, "Bang"), trailers);
RetryState state = CreateSimpleRetryState();
Assert.False(state.CanRetry(exception));
}
public async Task ResetTimeout()
{
var clock = new FakeClock(0);
var scheduler = new AdvanceFakeClockScheduler(clock);
var callSettings = CallSettings.FromExpiration(Expiration.FromTimeout(TimeSpan.FromSeconds(7)));
var state = new RetryState(clock, scheduler, s_retrySettings, callSettings);
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
// Reset should set the deadline of the call to CurrentTime + Timeout.
// That means that we can do two new retries without a timeout exception.
state.Reset();
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.WaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await Assert.ThrowsAsync <RpcException>(() => state.WaitAsync(exception, default));
// Verify that the clock has been advanced 12 seconds.
Assert.Equal(TimeSpan.FromSeconds(12).Ticks, clock.GetCurrentDateTimeUtc().Ticks);
}
public void CanRetry_ResourceExhausted_WithRetryInfo()
{
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(2))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception = new RpcException(new Status(StatusCode.ResourceExhausted, "Bang"), trailers);
RetryState state = CreateSimpleRetryState();
Assert.True(state.CanRetry(exception));
}
// See https://github.com/googleapis/google-cloud-java/blob/master/google-cloud-clients/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java#L2674
private async Task <PartialResultSet> ComputeNextAsync(CancellationToken cancellationToken)
{
// The retry state is local to the method as we're not trying to handle callers retrying.
RetryState retryState = new RetryState(_client.Settings.Clock ?? SystemClock.Instance, _client.Settings.Scheduler ?? SystemScheduler.Instance, _retrySettings, _callSettings);
while (true)
{
// If we've successfully read to the end of the stream and emptied the buffer, we've read all the responses.
if (_finished && _buffer.Count == 0)
{
return(null);
}
// Buffer contains items up to a resume token or has reached capacity: flush.
if (_buffer.Count > 0 && (_finished || !_safeToRetry || !_buffer.Last.Value.ResumeToken.IsEmpty))
{
var firstResult = _buffer.First.Value;
_buffer.RemoveFirst();
return(firstResult);
}
try
{
if (_grpcCall == null)
{
// Note: no cancellation token here; if we've been given a short cancellation token,
// it ought to apply to just the MoveNext call, not the original request.
_grpcCall = _request.ExecuteStreaming(_client, _callSettings);
}
bool hasNext = await _grpcCall.ResponseStream
.MoveNext(cancellationToken)
.WithSessionExpiryChecking(_session)
.ConfigureAwait(false);
retryState.Reset();
if (hasNext)
{
var next = _grpcCall.ResponseStream.Current;
var hasResumeToken = !next.ResumeToken.IsEmpty;
if (hasResumeToken)
{
_request.ResumeToken = next.ResumeToken;
_safeToRetry = true;
}
// If the buffer is empty and this result has a resume token or we cannot resume safely
// anyway, we can yield it immediately rather than placing it in the buffer to be
// returned on the next iteration.
if ((hasResumeToken || !_safeToRetry) && _buffer.Count == 0)
{
return(next);
}
_buffer.AddLast(next);
if (_buffer.Count > _maxBufferSize && !hasResumeToken)
{
// We need to flush without a restart token. Errors encountered until we see
// such a token will fail the read.
_safeToRetry = false;
}
}
else
{
_finished = true;
// Let the next iteration of the loop return 0 or buffered data.
}
}
catch (RpcException e) when(e.StatusCode == StatusCode.Cancelled && cancellationToken.IsCancellationRequested)
{
// gRPC throws RpcException, but it's more idiomatic to see an OperationCanceledException
cancellationToken.ThrowIfCancellationRequested();
}
catch (RpcException e) when(_safeToRetry && retryState.CanRetry(e))
{
_client.Settings.Logger.Warn($"Exception when reading from result stream. Retrying.", e);
await retryState.WaitAsync(e, cancellationToken).ConfigureAwait(false);
// Clear anything we've received since the previous response that contained a resume token
_buffer.Clear();
_grpcCall.Dispose();
_grpcCall = null;
}
}
}