public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default,
TimeSpan?timeout = null)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
await bucket.SendAsync(operation, token, timeout).ConfigureAwait(false);
break;
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.DurationValue.HasValue)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
break; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (TaskCanceledException) { ThrowTimeoutException(operation, timeout); }
catch (OperationCanceledException) { ThrowTimeoutException(operation, timeout); }
}
public async Task <T> RetryAsync <T>(Func <Task <T> > send, IRequest request) where T : IServiceResult
{
var token = request.Token;
if (token == CancellationToken.None)
{
var cts = new CancellationTokenSource(request.Timeout);
token = cts.Token;
}
//for measuring the capped duration
var stopwatch = new Stopwatch();
stopwatch.Start();
var backoff = ControlledBackoff.Create();
do
{
if (token.IsCancellationRequested)
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request);
}
try
{
var result = await send().ConfigureAwait(false);
var reason = result.RetryReason;
if (reason == RetryReason.NoRetry)
{
return(result);
}
if (reason.AlwaysRetry())
{
_logger.LogDebug(
"Retrying query {clientContextId}/{statement} because {reason}.", request.ClientContextId,
_redactor.UserData(request.Statement), reason);
await backoff.Delay(request).ConfigureAwait(false);
request.IncrementAttempts(reason);
continue;
}
var strategy = request.RetryStrategy;
var action = strategy.RetryAfter(request, reason);
if (action.Retry)
{
_logger.LogDebug(LoggingEvents.QueryEvent,
"Retrying query {clientContextId}/{statement} because {reason}.", request.ClientContextId,
request.Statement, reason);
var duration = action.DurationValue;
if (duration.HasValue)
{
_logger.LogDebug(
"Timeout for {clientContextId} is {timeout} and duration is {duration} and elapsed is {elapsed}",
request.ClientContextId, request.Timeout.TotalMilliseconds,
duration.Value.TotalMilliseconds, stopwatch.ElapsedMilliseconds);
var cappedDuration =
request.Timeout.CappedDuration(duration.Value, stopwatch.Elapsed);
_logger.LogDebug(
"Timeout for {clientContextId} capped duration is {duration} and elapsed is {elapsed}",
request.ClientContextId, cappedDuration.TotalMilliseconds,
stopwatch.ElapsedMilliseconds);
await Task.Delay(cappedDuration,
CancellationTokenSource.CreateLinkedTokenSource(token).Token).ConfigureAwait(false);
request.IncrementAttempts(reason);
//temp fix for query unit tests
if (request.Attempts > 9)
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request,
new InvalidOperationException($"Too many retries: {request.Attempts}."));
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request,
new InvalidOperationException($"Too many retries: {request.Attempts}."));
}
}
else
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request);
}
}
}
catch (TaskCanceledException _)
{
_logger.LogDebug("Request was canceled after {elapsed}.", stopwatch.ElapsedMilliseconds);
//timed out while waiting
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request, _);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request, _);
}
} while (true);
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
operation.Token = tokenPair;
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToString(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode
});
}
}
public async Task <T> RetryAsync <T>(Func <Task <T> > send, IRequest request) where T : IServiceResult
{
var token = request.Token;
CancellationTokenSource cts1 = null;
CancellationTokenSource cts2 = null;
if (request.Timeout > TimeSpan.Zero)
{
cts1 = CancellationTokenSourcePool.Shared.Rent(request.Timeout);
if (token.CanBeCanceled)
{
cts2 = CancellationTokenSource.CreateLinkedTokenSource(token, cts1.Token);
token = cts2.Token;
}
else
{
token = cts1.Token;
}
}
try
{
//for measuring the capped duration
var stopwatch = new Stopwatch();
stopwatch.Start();
var backoff = ControlledBackoff.Create();
do
{
if (token.IsCancellationRequested)
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request);
}
try
{
var result = await send().ConfigureAwait(false);
var reason = result.RetryReason;
if (reason == RetryReason.NoRetry)
{
return(result);
}
if (reason.AlwaysRetry())
{
LogRetryQuery(request.ClientContextId, _redactor.UserData(request.Statement), reason);
await backoff.Delay(request).ConfigureAwait(false);
request.IncrementAttempts(reason);
continue;
}
var strategy = request.RetryStrategy;
var action = strategy.RetryAfter(request, reason);
if (action.Retry)
{
LogRetryQuery(request.ClientContextId, _redactor.UserData(request.Statement), reason);
var duration = action.DurationValue;
if (duration.HasValue)
{
LogQueryDurations(request.ClientContextId, request.Timeout.TotalMilliseconds,
duration.Value.TotalMilliseconds, stopwatch.ElapsedMilliseconds);
var cappedDuration =
request.Timeout.CappedDuration(duration.Value, stopwatch.Elapsed);
LogCappedQueryDuration(request.ClientContextId, cappedDuration.TotalMilliseconds, stopwatch.ElapsedMilliseconds);
await Task.Delay(cappedDuration, token).ConfigureAwait(false);
request.IncrementAttempts(reason);
//temp fix for query unit tests
if (request.Attempts > 9)
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request,
new InvalidOperationException($"Too many retries: {request.Attempts}."));
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request,
new InvalidOperationException($"Too many retries: {request.Attempts}."));
}
}
else
{
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request);
}
}
else
{
//don't retry
result.ThrowOnNoRetry();
}
}
catch (TaskCanceledException _)
{
LogRequestCanceled(stopwatch.ElapsedMilliseconds);
//timed out while waiting
if (request.Idempotent)
{
UnambiguousTimeoutException.ThrowWithRetryReasons(request, _);
}
AmbiguousTimeoutException.ThrowWithRetryReasons(request, _);
}
} while (true);
}
finally
{
//stop recording metrics and either return result or throw exception
request.StopRecording();
cts2?.Dispose();
if (cts1 is not null)
{
CancellationTokenSourcePool.Shared.Return(cts1);
}
}
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
if (operation.Attempts > 1)
{
MetricTracker.KeyValue.TrackRetry(operation.OpCode);
}
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(operation is not GetCid &&
(e is ScopeNotFoundException ||
e is CollectionNotFoundException))
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
LogRefreshingCollectionId(e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
operation.IncrementAttempts(reason);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
operation.IncrementAttempts(reason);
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
MetricTracker.KeyValue.TrackTimeout(operation.OpCode);
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToStringInvariant(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode,
DispatchedFrom = operation.LastDispatchedFrom,
DispatchedTo = operation.LastDispatchedTo,
RetryReasons = operation.RetryReasons
});
}
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, token).ConfigureAwait(false);
break;
}
catch (CollectionOutdatedException e)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
token.ThrowIfCancellationRequested();
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
operation.Reset();
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
operation.Reset();
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (OperationCanceledException) { ThrowTimeoutException(operation, operation.Timeout); }
}
