// TODO: Move this retry code into production code, so that everyone can use it. private static T ExecuteWithRetryImpl <T>(Func <T> func) { Interlocked.Increment(ref _calls); // Make it easy to move this into production code later on by using IClock/IScheduler/IJitter var clock = SystemClock.Instance; var scheduler = SystemScheduler.Instance; var jitter = RetrySettings.RandomJitter; DateTime start = DateTime.UtcNow; DateTime end = start + s_timeout; // Immediate initial retry, before the exponential delay starts. TimeSpan retryDelay = TimeSpan.Zero; while (true) { try { return(func()); } catch (SpannerException e) when(e.IsRetryable) { TimeSpan actualDelay = jitter.GetDelay(retryDelay); DateTime expectedRetryTime = clock.GetCurrentDateTimeUtc() + actualDelay; if (expectedRetryTime > end) { throw; } scheduler.Sleep(actualDelay, CancellationToken.None); retryDelay = s_backoffSettings.NextDelay(retryDelay); Interlocked.Increment(ref _retries); } } }
/// <summary> /// Updates the state on the basis of the given exception, delaying for as long as is necessary /// between retries. /// </summary> internal async Task RecordErrorAndWaitAsync(RpcException exception, CancellationToken cancellationToken) { _consecutiveErrors++; TimeSpan thisDelay = GetRetryDelay(exception) ?? _nextDelay; await _scheduler.Delay(_backoffJitter.GetDelay(thisDelay), cancellationToken).ConfigureAwait(false); _nextDelay = _backoffSettings.NextDelay(_nextDelay); }
public void NextDelay(double current, double expectedNext) { var settings = new BackoffSettings( delay: TimeSpan.FromSeconds(1.0), maxDelay: TimeSpan.FromSeconds(5.0), delayMultiplier: 2.0); var expected = TimeSpan.FromSeconds(expectedNext); var actual = settings.NextDelay(TimeSpan.FromSeconds(current)); Assert.Equal(expected, actual); }
private async Task IncrementByOneAsync(SpannerConnection connection, bool orphanTransaction = false) { var backoffSettings = new BackoffSettings(TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(5), 1.5); TimeSpan nextDelay = TimeSpan.Zero; SpannerException spannerException; DateTime deadline = DateTime.UtcNow.AddSeconds(30); while (true) { spannerException = null; try { // We use manually created transactions here so the tests run on .NET Core. using (var transaction = await connection.BeginTransactionAsync()) { long current; using (var cmd = connection.CreateSelectCommand($"SELECT Int64Value FROM {_fixture.TableName} WHERE K=@k")) { cmd.Parameters.Add("k", SpannerDbType.String, _key); cmd.Transaction = transaction; var fetched = await cmd.ExecuteScalarAsync().ConfigureAwait(false); current = fetched is DBNull ? 0L : (long)fetched; } using (var cmd = connection.CreateUpdateCommand(_fixture.TableName)) { cmd.Parameters.Add("k", SpannerDbType.String, _key); cmd.Parameters.Add("Int64Value", SpannerDbType.Int64, current + 1); cmd.Transaction = transaction; await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); if (!orphanTransaction) { await transaction.CommitAsync().ConfigureAwait(false); } } } return; } // Keep trying for up to 30 seconds catch (SpannerException ex) when(ex.IsRetryable && DateTime.UtcNow < deadline) { nextDelay = backoffSettings.NextDelay(nextDelay); await Task.Delay(RetrySettings.RandomJitter.GetDelay(nextDelay)); spannerException = ex; } } }
internal async Task StartAsync() { // State used within the method. This is modified by local methods too. StreamInitializationCause cause = StreamInitializationCause.WatchStarting; FirestoreClient.ListenStream underlyingStream = null; TimeSpan nextBackoff = TimeSpan.Zero; try { // This won't actually run forever. Calling Stop will cancel the cancellation token, and we'll end up with // an exception which may or may not be caught. while (true) { var serverResponse = await GetNextResponse().ConfigureAwait(false); _callbackCancellationTokenSource.Token.ThrowIfCancellationRequested(); var result = await _state.HandleResponseAsync(serverResponse, _callbackCancellationTokenSource.Token).ConfigureAwait(false); switch (result) { case WatchResponseResult.Continue: break; case WatchResponseResult.ResetStream: await CloseStreamAsync().ConfigureAwait(false); cause = StreamInitializationCause.ResetRequested; break; case WatchResponseResult.StreamHealthy: nextBackoff = TimeSpan.Zero; break; default: throw new InvalidOperationException($"Unknown result type: {result}"); } // What about other exception types? } } // Swallow cancellation exceptions unless one of the user-provided cancellation tokens has been // cancelled, in which case it's fine to let it through. catch (OperationCanceledException) when(!_callbackCancellationTokenSource.Token.IsCancellationRequested) { // We really do just swallow the exception. No need for logging. } finally { lock (_stateLock) { _networkCancellationTokenSource.Dispose(); _callbackCancellationTokenSource.Dispose(); _stopCancellationTokenRegistration.Dispose(); _finished = true; } // Make sure we clean up even if we get an exception we don't handle explicitly. await CloseStreamAsync().ConfigureAwait(false); } // Local method responsible for fetching the next response from the server stream, including // stream initialization and error handling. async Task <ListenResponse> GetNextResponse() { while (true) { try { // If we're just starting, or we've closed the stream or it broke, restart. if (underlyingStream == null) { await _scheduler.Delay(_backoffJitter.GetDelay(nextBackoff), _networkCancellationTokenSource.Token).ConfigureAwait(false); nextBackoff = _backoffSettings.NextDelay(nextBackoff); underlyingStream = _db.Client.Listen(); await underlyingStream.TryWriteAsync(CreateRequest(_state.ResumeToken)).ConfigureAwait(false); _state.OnStreamInitialization(cause); } // Wait for a response or end-of-stream var next = await underlyingStream.ResponseStream.MoveNext(_networkCancellationTokenSource.Token).ConfigureAwait(false); // If the server provided a response, return it if (next) { return(underlyingStream.ResponseStream.Current); } // Otherwise, close the current stream and restart. await CloseStreamAsync().ConfigureAwait(false); cause = StreamInitializationCause.StreamCompleted; } catch (RpcException e) when(s_transientErrorStatusCodes.Contains(e.Status.StatusCode)) { // Close the current stream, ready to create a new one. await CloseStreamAsync().ConfigureAwait(false); // Extend the back-off if necessary. if (e.Status.StatusCode == StatusCode.ResourceExhausted) { nextBackoff = _backoffSettings.NextDelay(nextBackoff); } cause = StreamInitializationCause.RpcError; } } } async Task CloseStreamAsync() { if (underlyingStream != null) { var completeTask = underlyingStream.TryWriteCompleteAsync(); // TODO: Handle exceptions from this? if (completeTask != null) { await completeTask.ConfigureAwait(false); } underlyingStream.GrpcCall.Dispose(); } underlyingStream = null; } }