public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default,
TimeSpan?timeout = null)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
await bucket.SendAsync(operation, token, timeout).ConfigureAwait(false);
break;
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.DurationValue.HasValue)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
break; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (TaskCanceledException) { ThrowTimeoutException(operation, timeout); }
catch (OperationCanceledException) { ThrowTimeoutException(operation, timeout); }
}
private async Task <uint?> GetCidAsync(string fullyQualifiedName, bool sendAsBody, bool retryIfFailure)
{
using var rootSpan = RootSpan(OuterRequestSpans.ServiceSpan.Internal.GetCid);
using var getCid = new GetCid
{
Opaque = SequenceGenerator.GetNext(),
Span = rootSpan,
};
if (sendAsBody)
{
getCid.Content = fullyQualifiedName;
}
else
{
getCid.Key = fullyQualifiedName;
}
var options = new GetOptions();
_operationConfigurator.Configure(getCid, options.Transcoder(_rawStringTranscoder));
using var ctp = CreateRetryTimeoutCancellationTokenSource(options, getCid);
if (retryIfFailure)
{
await _bucket.RetryAsync(getCid, ctp.TokenPair).ConfigureAwait(false);
}
else
{
await _bucket.SendAsync(getCid, ctp.TokenPair).ConfigureAwait(false);
}
var resultWithValue = getCid.GetValueAsUint();
return(resultWithValue);
}
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions?options = null)
{
try
{
//sanity check for deferred bootstrapping errors
_bucket.ThrowIfBootStrapFailed();
options ??= new ExistsOptions();
using var getMetaOp = new GetMeta
{
Key = id,
Cid = Cid,
CName = Name,
Transcoder = _transcoder
};
await _bucket.SendAsync(getMetaOp, options.TokenValue, options.TimeoutValue).ConfigureAwait(false);
var result = getMetaOp.GetValue();
return(new ExistsResult
{
Cas = getMetaOp.Cas,
Exists = !result.Deleted
});
}
catch (DocumentNotFoundException)
{
return(new ExistsResult
{
Exists = false
});
}
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
operation.Token = tokenPair;
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToString(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode
});
}
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
if (operation.Attempts > 1)
{
MetricTracker.KeyValue.TrackRetry(operation.OpCode);
}
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(operation is not GetCid &&
(e is ScopeNotFoundException ||
e is CollectionNotFoundException))
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
LogRefreshingCollectionId(e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
operation.IncrementAttempts(reason);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
operation.IncrementAttempts(reason);
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
MetricTracker.KeyValue.TrackTimeout(operation.OpCode);
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToStringInvariant(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode,
DispatchedFrom = operation.LastDispatchedFrom,
DispatchedTo = operation.LastDispatchedTo,
RetryReasons = operation.RetryReasons
});
}
}
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions options)
{
using (var existsOp = new Observe
{
Key = id,
Cid = Cid,
Transcoder = _transcoder
})
{
try
{
await _bucket.SendAsync(existsOp, options.Token, options.Timeout);
var value = existsOp.GetValue();
return(new ExistsResult
{
Exists = existsOp.Success && value.KeyState != KeyState.NotFound &&
value.KeyState != KeyState.LogicalDeleted,
Cas = value.Cas,
Expiry = TimeSpan.FromMilliseconds(existsOp.Expires)
});
}
catch (KeyNotFoundException)
{
return(new ExistsResult
{
Exists = false
});
}
}
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, token).ConfigureAwait(false);
break;
}
catch (CollectionOutdatedException e)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
token.ThrowIfCancellationRequested();
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
operation.Reset();
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
operation.Reset();
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (OperationCanceledException) { ThrowTimeoutException(operation, operation.Timeout); }
}
public async Task <IExistsResult> ExistsAsync(string id, ExistsOptions?options = null)
{
//sanity check for deferred bootstrapping errors
_bucket.ThrowIfBootStrapFailed();
options ??= new ExistsOptions();
using var existsOp = new Observe
{
Key = id,
Cid = Cid,
CName = Name,
Transcoder = _transcoder
};
try
{
await _bucket.SendAsync(existsOp, options.TokenValue, options.TimeoutValue);
var value = existsOp.GetValue();
return(new ExistsResult
{
Exists = existsOp.Success && value.KeyState != KeyState.NotFound &&
value.KeyState != KeyState.LogicalDeleted,
Cas = value.Cas,
Expiry = TimeSpan.FromMilliseconds(existsOp.Expires)
});
}
catch (KeyNotFoundException)
{
return(new ExistsResult
{
Exists = false
});
}
}