public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default, TimeSpan?timeout = null) { try { var backoff = ControlledBackoff.Create(); do { try { operation.Attempts++; await bucket.SendAsync(operation, token, timeout).ConfigureAwait(false); break; } catch (CouchbaseException e) { if (e is IRetryable) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { if (token.IsCancellationRequested) { token.ThrowIfCancellationRequested(); } _logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.DurationValue.HasValue) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque, operation.Key, reason); await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false); } else { break; //don't retry } } else { throw; } } } while (true); } catch (TaskCanceledException) { ThrowTimeoutException(operation, timeout); } catch (OperationCanceledException) { ThrowTimeoutException(operation, timeout); } }
public async Task <T> RetryAsync <T>(Func <Task <T> > send, IRequest request) where T : IServiceResult { var token = request.Token; if (token == CancellationToken.None) { var cts = new CancellationTokenSource(request.Timeout); token = cts.Token; } //for measuring the capped duration var stopwatch = new Stopwatch(); stopwatch.Start(); var backoff = ControlledBackoff.Create(); do { if (token.IsCancellationRequested) { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request); } AmbiguousTimeoutException.ThrowWithRetryReasons(request); } try { var result = await send().ConfigureAwait(false); var reason = result.RetryReason; if (reason == RetryReason.NoRetry) { return(result); } if (reason.AlwaysRetry()) { _logger.LogDebug( "Retrying query {clientContextId}/{statement} because {reason}.", request.ClientContextId, _redactor.UserData(request.Statement), reason); await backoff.Delay(request).ConfigureAwait(false); request.IncrementAttempts(reason); continue; } var strategy = request.RetryStrategy; var action = strategy.RetryAfter(request, reason); if (action.Retry) { _logger.LogDebug(LoggingEvents.QueryEvent, "Retrying query {clientContextId}/{statement} because {reason}.", request.ClientContextId, request.Statement, reason); var duration = action.DurationValue; if (duration.HasValue) { _logger.LogDebug( "Timeout for {clientContextId} is {timeout} and duration is {duration} and elapsed is {elapsed}", request.ClientContextId, request.Timeout.TotalMilliseconds, duration.Value.TotalMilliseconds, stopwatch.ElapsedMilliseconds); var cappedDuration = request.Timeout.CappedDuration(duration.Value, stopwatch.Elapsed); _logger.LogDebug( "Timeout for {clientContextId} capped duration is {duration} and elapsed is {elapsed}", request.ClientContextId, cappedDuration.TotalMilliseconds, stopwatch.ElapsedMilliseconds); await Task.Delay(cappedDuration, CancellationTokenSource.CreateLinkedTokenSource(token).Token).ConfigureAwait(false); request.IncrementAttempts(reason); //temp fix for query unit tests if (request.Attempts > 9) { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request, new InvalidOperationException($"Too many retries: {request.Attempts}.")); } AmbiguousTimeoutException.ThrowWithRetryReasons(request, new InvalidOperationException($"Too many retries: {request.Attempts}.")); } } else { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request); } AmbiguousTimeoutException.ThrowWithRetryReasons(request); } } } catch (TaskCanceledException _) { _logger.LogDebug("Request was canceled after {elapsed}.", stopwatch.ElapsedMilliseconds); //timed out while waiting if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request, _); } AmbiguousTimeoutException.ThrowWithRetryReasons(request, _); } } while (true); }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default) { try { var backoff = ControlledBackoff.Create(); operation.Token = tokenPair; do { tokenPair.ThrowIfCancellationRequested(); try { operation.Attempts++; try { await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false); break; } catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. _logger.LogInformation("Updating stale manifest for collection and retrying.", e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); // no need to reset op in this case as it was not actually sent if (reason != RetryReason.CircuitBreakerOpen) { operation.Reset(); } continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.", operation.Opaque, operation.Key, reason); // Reset first so operation is not marked as sent if canceled during the delay operation.Reset(); await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair) .ConfigureAwait(false); } else { throw; //don't retry } } } while (true); } catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation) { ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext { BucketName = operation.BucketName, ClientContextId = operation.Opaque.ToString(), DocumentKey = operation.Key, Cas = operation.Cas, CollectionName = operation.CName, ScopeName = operation.SName, OpCode = operation.OpCode }); } }
public async Task <T> RetryAsync <T>(Func <Task <T> > send, IRequest request) where T : IServiceResult { var token = request.Token; CancellationTokenSource cts1 = null; CancellationTokenSource cts2 = null; if (request.Timeout > TimeSpan.Zero) { cts1 = CancellationTokenSourcePool.Shared.Rent(request.Timeout); if (token.CanBeCanceled) { cts2 = CancellationTokenSource.CreateLinkedTokenSource(token, cts1.Token); token = cts2.Token; } else { token = cts1.Token; } } try { //for measuring the capped duration var stopwatch = new Stopwatch(); stopwatch.Start(); var backoff = ControlledBackoff.Create(); do { if (token.IsCancellationRequested) { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request); } AmbiguousTimeoutException.ThrowWithRetryReasons(request); } try { var result = await send().ConfigureAwait(false); var reason = result.RetryReason; if (reason == RetryReason.NoRetry) { return(result); } if (reason.AlwaysRetry()) { LogRetryQuery(request.ClientContextId, _redactor.UserData(request.Statement), reason); await backoff.Delay(request).ConfigureAwait(false); request.IncrementAttempts(reason); continue; } var strategy = request.RetryStrategy; var action = strategy.RetryAfter(request, reason); if (action.Retry) { LogRetryQuery(request.ClientContextId, _redactor.UserData(request.Statement), reason); var duration = action.DurationValue; if (duration.HasValue) { LogQueryDurations(request.ClientContextId, request.Timeout.TotalMilliseconds, duration.Value.TotalMilliseconds, stopwatch.ElapsedMilliseconds); var cappedDuration = request.Timeout.CappedDuration(duration.Value, stopwatch.Elapsed); LogCappedQueryDuration(request.ClientContextId, cappedDuration.TotalMilliseconds, stopwatch.ElapsedMilliseconds); await Task.Delay(cappedDuration, token).ConfigureAwait(false); request.IncrementAttempts(reason); //temp fix for query unit tests if (request.Attempts > 9) { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request, new InvalidOperationException($"Too many retries: {request.Attempts}.")); } AmbiguousTimeoutException.ThrowWithRetryReasons(request, new InvalidOperationException($"Too many retries: {request.Attempts}.")); } } else { if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request); } AmbiguousTimeoutException.ThrowWithRetryReasons(request); } } else { //don't retry result.ThrowOnNoRetry(); } } catch (TaskCanceledException _) { LogRequestCanceled(stopwatch.ElapsedMilliseconds); //timed out while waiting if (request.Idempotent) { UnambiguousTimeoutException.ThrowWithRetryReasons(request, _); } AmbiguousTimeoutException.ThrowWithRetryReasons(request, _); } } while (true); } finally { //stop recording metrics and either return result or throw exception request.StopRecording(); cts2?.Dispose(); if (cts1 is not null) { CancellationTokenSourcePool.Shared.Return(cts1); } } }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default) { try { var backoff = ControlledBackoff.Create(); do { tokenPair.ThrowIfCancellationRequested(); try { if (operation.Attempts > 1) { MetricTracker.KeyValue.TrackRetry(operation.OpCode); } try { await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false); break; } catch (CouchbaseException e) when(operation is not GetCid && (e is ScopeNotFoundException || e is CollectionNotFoundException)) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. LogRefreshingCollectionId(e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason); await backoff.Delay(operation).ConfigureAwait(false); // no need to reset op in this case as it was not actually sent if (reason != RetryReason.CircuitBreakerOpen) { operation.Reset(); } operation.IncrementAttempts(reason); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason); // Reset first so operation is not marked as sent if canceled during the delay operation.Reset(); operation.IncrementAttempts(reason); await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair) .ConfigureAwait(false); } else { throw; //don't retry } } } while (true); } catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation) { MetricTracker.KeyValue.TrackTimeout(operation.OpCode); ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext { BucketName = operation.BucketName, ClientContextId = operation.Opaque.ToStringInvariant(), DocumentKey = operation.Key, Cas = operation.Cas, CollectionName = operation.CName, ScopeName = operation.SName, OpCode = operation.OpCode, DispatchedFrom = operation.LastDispatchedFrom, DispatchedTo = operation.LastDispatchedTo, RetryReasons = operation.RetryReasons }); } }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default) { try { var backoff = ControlledBackoff.Create(); do { try { operation.Attempts++; try { await bucket.SendAsync(operation, token).ConfigureAwait(false); break; } catch (CollectionOutdatedException e) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. _logger.LogInformation("Updating stale manifest for collection and retrying.", e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) { if (e is IRetryable) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { token.ThrowIfCancellationRequested(); _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); operation.Reset(); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.", operation.Opaque, operation.Key, reason); operation.Reset(); await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false); } else { throw; //don't retry } } else { throw; } } } while (true); } catch (OperationCanceledException) { ThrowTimeoutException(operation, operation.Timeout); } }