// TODO: Move this retry code into production code, so that everyone can use it.
private static T ExecuteWithRetryImpl <T>(Func <T> func)
{
Interlocked.Increment(ref _calls);
// Make it easy to move this into production code later on by using IClock/IScheduler/IJitter
var clock = SystemClock.Instance;
var scheduler = SystemScheduler.Instance;
var jitter = RetrySettings.RandomJitter;
DateTime start = DateTime.UtcNow;
DateTime end = start + s_timeout;
// Immediate initial retry, before the exponential delay starts.
TimeSpan retryDelay = TimeSpan.Zero;
while (true)
{
try
{
return(func());
}
catch (SpannerException e) when(e.IsRetryable)
{
TimeSpan actualDelay = jitter.GetDelay(retryDelay);
DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay;
if (expectedRetryTime > end)
{
throw;
}
scheduler.Sleep(actualDelay, CancellationToken.None);
retryDelay = s_backoffSettings.NextDelay(retryDelay);
Interlocked.Increment(ref _retries);
}
}
}
/// <summary>
/// Updates the state on the basis of the given exception, delaying for as long as is necessary
/// between retries.
/// </summary>
internal async Task RecordErrorAndWaitAsync(RpcException exception, CancellationToken cancellationToken)
{
_consecutiveErrors++;
TimeSpan thisDelay = GetRetryDelay(exception) ?? _nextDelay;
await _scheduler.Delay(_backoffJitter.GetDelay(thisDelay), cancellationToken).ConfigureAwait(false);
_nextDelay = _backoffSettings.NextDelay(_nextDelay);
}
public void NextDelay(double current, double expectedNext)
{
var settings = new BackoffSettings(
delay: TimeSpan.FromSeconds(1.0),
maxDelay: TimeSpan.FromSeconds(5.0),
delayMultiplier: 2.0);
var expected = TimeSpan.FromSeconds(expectedNext);
var actual = settings.NextDelay(TimeSpan.FromSeconds(current));
Assert.Equal(expected, actual);
}
private async Task IncrementByOneAsync(SpannerConnection connection, bool orphanTransaction = false)
{
var backoffSettings = new BackoffSettings(TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(5), 1.5);
TimeSpan nextDelay = TimeSpan.Zero;
SpannerException spannerException;
DateTime deadline = DateTime.UtcNow.AddSeconds(30);
while (true)
{
spannerException = null;
try
{
// We use manually created transactions here so the tests run on .NET Core.
using (var transaction = await connection.BeginTransactionAsync())
{
long current;
using (var cmd = connection.CreateSelectCommand($"SELECT Int64Value FROM {_fixture.TableName} WHERE K=@k"))
{
cmd.Parameters.Add("k", SpannerDbType.String, _key);
cmd.Transaction = transaction;
var fetched = await cmd.ExecuteScalarAsync().ConfigureAwait(false);
current = fetched is DBNull ? 0L : (long)fetched;
}
using (var cmd = connection.CreateUpdateCommand(_fixture.TableName))
{
cmd.Parameters.Add("k", SpannerDbType.String, _key);
cmd.Parameters.Add("Int64Value", SpannerDbType.Int64, current + 1);
cmd.Transaction = transaction;
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
if (!orphanTransaction)
{
await transaction.CommitAsync().ConfigureAwait(false);
}
}
}
return;
}
// Keep trying for up to 30 seconds
catch (SpannerException ex) when(ex.IsRetryable && DateTime.UtcNow < deadline)
{
nextDelay = backoffSettings.NextDelay(nextDelay);
await Task.Delay(RetrySettings.RandomJitter.GetDelay(nextDelay));
spannerException = ex;
}
}
}
internal async Task StartAsync()
{
// State used within the method. This is modified by local methods too.
StreamInitializationCause cause = StreamInitializationCause.WatchStarting;
FirestoreClient.ListenStream underlyingStream = null;
TimeSpan nextBackoff = TimeSpan.Zero;
try
{
// This won't actually run forever. Calling Stop will cancel the cancellation token, and we'll end up with
// an exception which may or may not be caught.
while (true)
{
var serverResponse = await GetNextResponse().ConfigureAwait(false);
_callbackCancellationTokenSource.Token.ThrowIfCancellationRequested();
var result = await _state.HandleResponseAsync(serverResponse, _callbackCancellationTokenSource.Token).ConfigureAwait(false);
switch (result)
{
case WatchResponseResult.Continue:
break;
case WatchResponseResult.ResetStream:
await CloseStreamAsync().ConfigureAwait(false);
cause = StreamInitializationCause.ResetRequested;
break;
case WatchResponseResult.StreamHealthy:
nextBackoff = TimeSpan.Zero;
break;
default:
throw new InvalidOperationException($"Unknown result type: {result}");
}
// What about other exception types?
}
}
// Swallow cancellation exceptions unless one of the user-provided cancellation tokens has been
// cancelled, in which case it's fine to let it through.
catch (OperationCanceledException) when(!_callbackCancellationTokenSource.Token.IsCancellationRequested)
{
// We really do just swallow the exception. No need for logging.
}
finally
{
lock (_stateLock)
{
_networkCancellationTokenSource.Dispose();
_callbackCancellationTokenSource.Dispose();
_stopCancellationTokenRegistration.Dispose();
_finished = true;
}
// Make sure we clean up even if we get an exception we don't handle explicitly.
await CloseStreamAsync().ConfigureAwait(false);
}
// Local method responsible for fetching the next response from the server stream, including
// stream initialization and error handling.
async Task <ListenResponse> GetNextResponse()
{
while (true)
{
try
{
// If we're just starting, or we've closed the stream or it broke, restart.
if (underlyingStream == null)
{
await _scheduler.Delay(_backoffJitter.GetDelay(nextBackoff), _networkCancellationTokenSource.Token).ConfigureAwait(false);
nextBackoff = _backoffSettings.NextDelay(nextBackoff);
underlyingStream = _db.Client.Listen();
await underlyingStream.TryWriteAsync(CreateRequest(_state.ResumeToken)).ConfigureAwait(false);
_state.OnStreamInitialization(cause);
}
// Wait for a response or end-of-stream
var next = await underlyingStream.ResponseStream.MoveNext(_networkCancellationTokenSource.Token).ConfigureAwait(false);
// If the server provided a response, return it
if (next)
{
return(underlyingStream.ResponseStream.Current);
}
// Otherwise, close the current stream and restart.
await CloseStreamAsync().ConfigureAwait(false);
cause = StreamInitializationCause.StreamCompleted;
}
catch (RpcException e) when(s_transientErrorStatusCodes.Contains(e.Status.StatusCode))
{
// Close the current stream, ready to create a new one.
await CloseStreamAsync().ConfigureAwait(false);
// Extend the back-off if necessary.
if (e.Status.StatusCode == StatusCode.ResourceExhausted)
{
nextBackoff = _backoffSettings.NextDelay(nextBackoff);
}
cause = StreamInitializationCause.RpcError;
}
}
}
async Task CloseStreamAsync()
{
if (underlyingStream != null)
{
var completeTask = underlyingStream.TryWriteCompleteAsync();
// TODO: Handle exceptions from this?
if (completeTask != null)
{
await completeTask.ConfigureAwait(false);
}
underlyingStream.GrpcCall.Dispose();
}
underlyingStream = null;
}
}