public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default, TimeSpan?timeout = null) { try { var backoff = ControlledBackoff.Create(); do { try { operation.Attempts++; await bucket.SendAsync(operation, token, timeout).ConfigureAwait(false); break; } catch (CouchbaseException e) { if (e is IRetryable) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { if (token.IsCancellationRequested) { token.ThrowIfCancellationRequested(); } _logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.DurationValue.HasValue) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque, operation.Key, reason); await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false); } else { break; //don't retry } } else { throw; } } } while (true); } catch (TaskCanceledException) { ThrowTimeoutException(operation, timeout); } catch (OperationCanceledException) { ThrowTimeoutException(operation, timeout); } }
private async Task <uint?> GetCidAsync(string fullyQualifiedName, bool sendAsBody, bool retryIfFailure) { using var rootSpan = RootSpan(OuterRequestSpans.ServiceSpan.Internal.GetCid); using var getCid = new GetCid { Opaque = SequenceGenerator.GetNext(), Span = rootSpan, }; if (sendAsBody) { getCid.Content = fullyQualifiedName; } else { getCid.Key = fullyQualifiedName; } var options = new GetOptions(); _operationConfigurator.Configure(getCid, options.Transcoder(_rawStringTranscoder)); using var ctp = CreateRetryTimeoutCancellationTokenSource(options, getCid); if (retryIfFailure) { await _bucket.RetryAsync(getCid, ctp.TokenPair).ConfigureAwait(false); } else { await _bucket.SendAsync(getCid, ctp.TokenPair).ConfigureAwait(false); } var resultWithValue = getCid.GetValueAsUint(); return(resultWithValue); }
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions?options = null) { try { //sanity check for deferred bootstrapping errors _bucket.ThrowIfBootStrapFailed(); options ??= new ExistsOptions(); using var getMetaOp = new GetMeta { Key = id, Cid = Cid, CName = Name, Transcoder = _transcoder }; await _bucket.SendAsync(getMetaOp, options.TokenValue, options.TimeoutValue).ConfigureAwait(false); var result = getMetaOp.GetValue(); return(new ExistsResult { Cas = getMetaOp.Cas, Exists = !result.Deleted }); } catch (DocumentNotFoundException) { return(new ExistsResult { Exists = false }); } }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default) { try { var backoff = ControlledBackoff.Create(); operation.Token = tokenPair; do { tokenPair.ThrowIfCancellationRequested(); try { operation.Attempts++; try { await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false); break; } catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. _logger.LogInformation("Updating stale manifest for collection and retrying.", e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); // no need to reset op in this case as it was not actually sent if (reason != RetryReason.CircuitBreakerOpen) { operation.Reset(); } continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.", operation.Opaque, operation.Key, reason); // Reset first so operation is not marked as sent if canceled during the delay operation.Reset(); await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair) .ConfigureAwait(false); } else { throw; //don't retry } } } while (true); } catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation) { ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext { BucketName = operation.BucketName, ClientContextId = operation.Opaque.ToString(), DocumentKey = operation.Key, Cas = operation.Cas, CollectionName = operation.CName, ScopeName = operation.SName, OpCode = operation.OpCode }); } }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default) { try { var backoff = ControlledBackoff.Create(); do { tokenPair.ThrowIfCancellationRequested(); try { if (operation.Attempts > 1) { MetricTracker.KeyValue.TrackRetry(operation.OpCode); } try { await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false); break; } catch (CouchbaseException e) when(operation is not GetCid && (e is ScopeNotFoundException || e is CollectionNotFoundException)) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. LogRefreshingCollectionId(e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason); await backoff.Delay(operation).ConfigureAwait(false); // no need to reset op in this case as it was not actually sent if (reason != RetryReason.CircuitBreakerOpen) { operation.Reset(); } operation.IncrementAttempts(reason); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason); // Reset first so operation is not marked as sent if canceled during the delay operation.Reset(); operation.IncrementAttempts(reason); await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair) .ConfigureAwait(false); } else { throw; //don't retry } } } while (true); } catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation) { MetricTracker.KeyValue.TrackTimeout(operation.OpCode); ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext { BucketName = operation.BucketName, ClientContextId = operation.Opaque.ToStringInvariant(), DocumentKey = operation.Key, Cas = operation.Cas, CollectionName = operation.CName, ScopeName = operation.SName, OpCode = operation.OpCode, DispatchedFrom = operation.LastDispatchedFrom, DispatchedTo = operation.LastDispatchedTo, RetryReasons = operation.RetryReasons }); } }
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions options) { using (var existsOp = new Observe { Key = id, Cid = Cid, Transcoder = _transcoder }) { try { await _bucket.SendAsync(existsOp, options.Token, options.Timeout); var value = existsOp.GetValue(); return(new ExistsResult { Exists = existsOp.Success && value.KeyState != KeyState.NotFound && value.KeyState != KeyState.LogicalDeleted, Cas = value.Cas, Expiry = TimeSpan.FromMilliseconds(existsOp.Expires) }); } catch (KeyNotFoundException) { return(new ExistsResult { Exists = false }); } } }
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default) { try { var backoff = ControlledBackoff.Create(); do { try { operation.Attempts++; try { await bucket.SendAsync(operation, token).ConfigureAwait(false); break; } catch (CollectionOutdatedException e) { // We catch CollectionOutdatedException separately from the CouchbaseException catch block // in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic. _logger.LogInformation("Updating stale manifest for collection and retrying.", e); if (!await RefreshCollectionId(bucket, operation) .ConfigureAwait(false)) { // rethrow if we fail to refresh he collection ID so we hit retry logic // otherwise we'll loop and retry immediately throw; } } } catch (CouchbaseException e) { if (e is IRetryable) { var reason = e.ResolveRetryReason(); if (reason.AlwaysRetry()) { token.ThrowIfCancellationRequested(); _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.", operation.Opaque, operation.Key, reason); await backoff.Delay(operation).ConfigureAwait(false); operation.Reset(); continue; } var strategy = operation.RetryStrategy; var action = strategy.RetryAfter(operation, reason); if (action.Retry) { _logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.", operation.Opaque, operation.Key, reason); operation.Reset(); await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false); } else { throw; //don't retry } } else { throw; } } } while (true); } catch (OperationCanceledException) { ThrowTimeoutException(operation, operation.Timeout); } }
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions?options = null) { //sanity check for deferred bootstrapping errors _bucket.ThrowIfBootStrapFailed(); options ??= new ExistsOptions(); using var existsOp = new Observe { Key = id, Cid = Cid, CName = Name, Transcoder = _transcoder }; try { await _bucket.SendAsync(existsOp, options.TokenValue, options.TimeoutValue); var value = existsOp.GetValue(); return(new ExistsResult { Exists = existsOp.Success && value.KeyState != KeyState.NotFound && value.KeyState != KeyState.LogicalDeleted, Cas = value.Cas, Expiry = TimeSpan.FromMilliseconds(existsOp.Expires) }); } catch (KeyNotFoundException) { return(new ExistsResult { Exists = false }); } }