public async Task RecordErrorAndWait_RetryInfo()
{
var settings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
// Delay taken from retry info
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(3), default)).Returns(Task.FromResult(0));
// Delay taken from backoff settings (which have still doubled, even when the first value wasn't used)
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
// The first exception contains retry info, so we don't use the backoff settings
var retryInfo = new Rpc.RetryInfo {
RetryDelay = Duration.FromTimeSpan(TimeSpan.FromSeconds(3))
};
Metadata trailers = new Metadata
{
{ RetryState.RetryInfoKey, retryInfo.ToByteArray() }
};
var exception1 = new RpcException(new Status(StatusCode.Unavailable, "Bang"), trailers);
var exception2 = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
RetryState state = new RetryState(mock.Object, settings, RetrySettings.NoJitter, maxConsecutiveErrors: 5);
Assert.True(state.CanRetry(exception1));
await state.RecordErrorAndWaitAsync(exception1, default);
Assert.True(state.CanRetry(exception2));
await state.RecordErrorAndWaitAsync(exception2, default);
}
public async Task RecordErrorAndWait_BackoffSettingsObeyed()
{
var settings = new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(4), default)).Returns(Task.FromResult(0));
// Retry maxes out at 5 seconds
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(5), default)).Returns(Task.FromResult(0));
// After reset
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
var state = new RetryState(mock.Object, settings, RetrySettings.NoJitter, maxConsecutiveErrors: 5);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
state.Reset();
await state.RecordErrorAndWaitAsync(exception, default);
}
public async Task RecordErrorAndWait_RetrySettingsObeyed()
{
RetrySettings retrySettings = RetrySettings.FromExponentialBackoff(
maxAttempts: int.MaxValue, // Ignored in SqlResultStream
initialBackoff: TimeSpan.FromSeconds(1),
maxBackoff: TimeSpan.FromSeconds(5),
backoffMultiplier: 2.0,
ignored => false, // Ignored in SqlResultStream
RetrySettings.NoJitter);
var mock = new Mock <IScheduler>(MockBehavior.Strict);
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(2), default)).Returns(Task.FromResult(0));
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(4), default)).Returns(Task.FromResult(0));
// Retry maxes out at 5 seconds
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(5), default)).Returns(Task.FromResult(0));
// After reset
mock.Setup(s => s.Delay(TimeSpan.FromSeconds(1), default)).Returns(Task.FromResult(0));
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
var state = new RetryState(mock.Object, retrySettings, maxConsecutiveErrors: 5);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
await state.RecordErrorAndWaitAsync(exception, default);
state.Reset();
await state.RecordErrorAndWaitAsync(exception, default);
}
public async Task CanRetry_MaxConsecutiveRetries_NoReset()
{
var state = new RetryState(
new NoOpScheduler(),
s_retrySettings,
maxConsecutiveErrors: 2);
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.False(state.CanRetry(exception));
}
public async Task CanRetry_MaxConsecutiveRetries_NoReset()
{
var state = new RetryState(
new NoOpScheduler(),
new BackoffSettings(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(15), 2.0),
RetrySettings.NoJitter,
maxConsecutiveErrors: 2);
var exception = new RpcException(new Status(StatusCode.Unavailable, "Bang"));
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.True(state.CanRetry(exception));
await state.RecordErrorAndWaitAsync(exception, default);
Assert.False(state.CanRetry(exception));
}
// See https://github.com/googleapis/google-cloud-java/blob/master/google-cloud-clients/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java#L2674
private async Task <PartialResultSet> ComputeNextAsync(CancellationToken cancellationToken)
{
// The retry state is local to the method as we're not trying to handle callers retrying.
RetryState retryState = new RetryState(_client.Settings.Scheduler ?? SystemScheduler.Instance, _backoffSettings, _backoffJitter);
while (true)
{
// If we've successfully read to the end of the stream and emptied the buffer, we've read all the responses.
if (_finished && _buffer.Count == 0)
{
return(null);
}
// Buffer contains items up to a resume token or has reached capacity: flush.
if (_buffer.Count > 0 && (_finished || !_safeToRetry || !_buffer.Last.Value.ResumeToken.IsEmpty))
{
var firstResult = _buffer.First.Value;
_buffer.RemoveFirst();
return(firstResult);
}
try
{
if (_grpcCall == null)
{
// Note: no cancellation token here; if we've been given a short cancellation token,
// it ought to apply to just the MoveNext call, not the original request.
_grpcCall = _client.ExecuteStreamingSql(_request, _callSettings).GrpcCall;
}
bool hasNext = await _grpcCall.ResponseStream
.MoveNext(cancellationToken)
.WithSessionExpiryChecking(_session)
.ConfigureAwait(false);
retryState.Reset();
if (hasNext)
{
var next = _grpcCall.ResponseStream.Current;
var hasResumeToken = !next.ResumeToken.IsEmpty;
if (hasResumeToken)
{
_request.ResumeToken = next.ResumeToken;
_safeToRetry = true;
}
// If the buffer is empty and this result has a resume token or we cannot resume safely
// anyway, we can yield it immediately rather than placing it in the buffer to be
// returned on the next iteration.
if ((hasResumeToken || !_safeToRetry) && _buffer.Count == 0)
{
return(next);
}
_buffer.AddLast(next);
if (_buffer.Count > _maxBufferSize && !hasResumeToken)
{
// We need to flush without a restart token. Errors encountered until we see
// such a token will fail the read.
_safeToRetry = false;
}
}
else
{
_finished = true;
// Let the next iteration of the loop return 0 or buffered data.
}
}
catch (RpcException e) when(e.StatusCode == StatusCode.Cancelled && cancellationToken.IsCancellationRequested)
{
// gRPC throws RpcException, but it's more idiomatic to see an OperationCanceledException
cancellationToken.ThrowIfCancellationRequested();
}
catch (RpcException e) when(_safeToRetry && retryState.CanRetry(e))
{
_client.Settings.Logger.Warn($"Exception when reading from result stream. Retrying.", e);
await retryState.RecordErrorAndWaitAsync(e, cancellationToken).ConfigureAwait(false);
// Clear anything we've received since the previous response that contained a resume token
_buffer.Clear();
_grpcCall.Dispose();
_grpcCall = null;
}
}
}