public virtual Task MockableRetryAsync(IOperation operation, CancellationTokenPair cancellationToken)
{
// BucketBase.RetryAsync forwards to our mock IRetryOrchestrator, which forwards back to this method
// This allows us to mock retry behavior, while BucketBase.RetryAsync remains non-virtual for performance.
return(Task.CompletedTask);
}
protected sealed override TaskResult ExecuteTask(CancellationTokenPair token)
{
Dry(token);
using var state = TaskResult.SpinLock(TimeSpan.FromMilliseconds(500));
state.Value = state.Value.WithTerminationTaskResultType(TaskResultCode.SuccessResult);
return(state.Value);
}
private async Task AssertRetryThenSuccessAsync(OperationBase op, Exception exp)
{
var retryOrchestrator = CreateRetryOrchestrator();
var bucketMock = new Mock <BucketBase>("fake", new ClusterContext(), new Mock <IScopeFactory>().Object,
retryOrchestrator, new Mock <ILogger>().Object, new Mock <IRedactor>().Object,
new Mock <IBootstrapperFactory>().Object, NoopRequestTracer.Instance, new Mock <IOperationConfigurator>().Object,
new BestEffortRetryStrategy());
bucketMock.Setup(x => x.SendAsync(op, It.IsAny <CancellationTokenPair>())).Callback((IOperation op, CancellationTokenPair ct) =>
{
if (op.Completed.IsCompleted)
{
Assert.True(false, "operation result should be reset before retry");
}
// complete the operation (ResponseStatus does not matter for this test)
op.HandleOperationCompleted(AsyncState.BuildErrorResponse(op.Opaque, ResponseStatus.TemporaryFailure));
if (op.Attempts == 1)
{
throw exp;
}
}).Returns(op.Completed.AsTask());
var tokenSource = new CancellationTokenSource(TimeSpan.FromMilliseconds(2500));
try
{
await retryOrchestrator.RetryAsync(bucketMock.Object, op, CancellationTokenPair.FromInternalToken(tokenSource.Token)).ConfigureAwait(false);
}
catch (Exception)
{
Assert.True(false, "Expected operation to succeed after retry");
}
Assert.True(op.Attempts > 1);
}
private async Task AssertRetryAsync <TExpected>(IOperation op, Exception exp, int minAttempts = 2,
CancellationToken externalCancellationToken = default)
where TExpected : Exception
{
var retryOrchestrator = CreateRetryOrchestrator();
var bucketMock = new Mock <BucketBase>("fake", new ClusterContext(), new Mock <IScopeFactory>().Object,
retryOrchestrator, new Mock <ILogger>().Object, new Mock <IRedactor>().Object,
new Mock <IBootstrapperFactory>().Object, NoopRequestTracer.Instance,
new Mock <IOperationConfigurator>().Object,
new BestEffortRetryStrategy());
bucketMock.Setup(x => x.SendAsync(op, It.IsAny <CancellationTokenPair>())).Throws(exp);
using var tokenSource = new CancellationTokenSource(TimeSpan.FromMilliseconds(25000));
try
{
var tokenPair = new CancellationTokenPair(externalCancellationToken, tokenSource.Token);
await retryOrchestrator.RetryAsync(bucketMock.Object, op, tokenPair).ConfigureAwait(false);
}
catch (Exception e)
{
Assert.IsAssignableFrom <TExpected>(e);
Assert.True(op.Attempts >= minAttempts);
}
}
internal override async Task SendAsync(IOperation op, CancellationTokenPair token = default)
{
var mockConnectionPool = new Mock <IConnectionPool>();
var mockConnectionPoolFactory = new Mock <IConnectionPoolFactory>();
mockConnectionPoolFactory
.Setup(m => m.Create(It.IsAny <ClusterNode>()))
.Returns(mockConnectionPool.Object);
var clusterNode = new ClusterNode(new ClusterContext(), mockConnectionPoolFactory.Object,
new Mock <ILogger <ClusterNode> >().Object, new DefaultObjectPool <OperationBuilder>(new OperationBuilderPoolPolicy()),
new Mock <ICircuitBreaker>().Object,
new Mock <ISaslMechanismFactory>().Object,
new Mock <IRedactor>().Object,
new IPEndPoint(IPAddress.Parse("127.0.0.1"), 11210),
BucketType.Couchbase,
new NodeAdapter(),
NoopRequestTracer.Instance);
await clusterNode.ExecuteOp(op, token).ConfigureAwait(false);
if (_statuses.TryDequeue(out ResponseStatus status))
{
(op as OperationBase)?.HandleOperationCompleted(AsyncState.BuildErrorResponse(0, status));
}
else
{
throw new InvalidOperationException();
}
}
private CancellationTokenSource CreateRetryTimeoutCancellationTokenSource(
FunctionOptionsBase options, out CancellationTokenPair tokenPair)
{
var cts = new CancellationTokenSource(options.Timeout);
tokenPair = new CancellationTokenPair(options.Token, cts.Token);
return(cts);
}
private CancellationTokenSource CreateRetryTimeoutCancellationTokenSource(
ITimeoutOptions options, IOperation op, out CancellationTokenPair tokenPair)
{
var cts = new CancellationTokenSource(GetTimeout(options.Timeout, op));
tokenPair = new CancellationTokenPair(options.Token, cts.Token);
return(cts);
}
protected virtual void Dry(CancellationTokenPair pair)
{
byte unitsToRemove;
{
using var lls = LaundryFlags.SpinLock(TimeSpan.FromSeconds(2));
unitsToRemove = lls.LoadedLaundryItem != LaundryItems.InvalidItem
? lls.LoadedLaundryItem.Dampness
: throw new StateLogicErrorException(
"It should not be possible to lack laundry during the wash cycle.");
}
byte unitsToDeduct = unitsToRemove;
TimeSpan timeRequired = TimeToDecrementDampnessPerUnit * unitsToRemove;
double randomFactor = RandomNumberSource.Next(1, 11) / 100.0; //introduce randomness factor of 10%, either direction
Debug.Assert(randomFactor >= 0.0 && randomFactor <= 0.11);
TimeSpan randomTimeToAddOrSub = timeRequired * randomFactor;
Debug.Assert(randomTimeToAddOrSub <= timeRequired);
bool negate = RandomNumberSource.Next(1, 3) == 1;
randomTimeToAddOrSub = negate ? -randomTimeToAddOrSub : +randomTimeToAddOrSub;
timeRequired += randomTimeToAddOrSub;
Stopwatch sw = null;
try
{
sw = HighPrecisionTimer;
sw.Restart();
SimulateWait(pair, timeRequired);
}
catch (OperationCanceledException)
{
if (sw == null)
{
string log = "For some reason the stopwatch is null.";
TerminationHelper.TerminateApplication(log);
return;
}
TimeSpan elapsed = sw.Elapsed;
double percentage = elapsed >= timeRequired ? 1.0 : elapsed / timeRequired;
unitsToDeduct = Convert.ToByte(Math.Floor(unitsToRemove * percentage));
throw;
}
finally
{
sw?.Reset();
byte newSetting = (byte)(unitsToRemove - unitsToDeduct);
Debug.Assert(newSetting <= unitsToRemove);
using var lls = LaundryFlags.SpinLock(TimeSpan.FromSeconds(2));
lls.SetDampFactor(newSetting);
Debug.Assert(lls.LoadedLaundryItem.Dampness <= unitsToRemove);
}
}
public void CanBeCanceled_SameToken_True()
{
// Arrange
using var cts = new CancellationTokenSource();
var tokenPair = new CancellationTokenPair(cts.Token, cts.Token);
// Assert
Assert.True(tokenPair.CanBeCanceled);
}
public void CanBeCanceled_InternalToken_True()
{
// Arrange
using var cts = new CancellationTokenSource();
var tokenPair = CancellationTokenPair.FromInternalToken(cts.Token);
// Assert
Assert.True(tokenPair.CanBeCanceled);
}
public void IsCancellationRequested_TokensNotCanceled_False()
{
// Arrange
using var cts = new CancellationTokenSource();
using var cts2 = new CancellationTokenSource();
var tokenPair = new CancellationTokenPair(cts.Token, cts2.Token);
// Assert
Assert.False(tokenPair.IsCancellationRequested);
}
protected virtual void Soak(CancellationTokenPair token)
{
Stopwatch sw = null;
try
{
Debug.WriteLine("Beginning Cleanse Soak");
byte oldDampness;
byte newDampness;
sw = HighPrecisionTimer;
sw.Reset();
sw.Start();
{
using LockedLaundryStatus lls =
LaundryFlags.SpinLock(token.IndividualToken, TimeSpan.FromSeconds(2));
var res = lls.SoakLaundry() ??
throw new StateLogicErrorException(
"It is supposed to be impossible to start the machine without laundry in it.");
oldDampness = res.OldDampness;
newDampness = res.NewDampness;
}
Debug.Assert(newDampness >= oldDampness);
int dampnessUnitsIncrease = newDampness - oldDampness;
TimeSpan totalTimeRequired = TimeToIncrementDampnessPerUnit * dampnessUnitsIncrease;
TimeSpan timeRemaining = totalTimeRequired - sw.Elapsed;
if (timeRemaining > TimeSpan.Zero)
{
SimulateWait(token, timeRemaining, "Beginning soak wait", "Ending soak wait");
}
}
catch (StateLogicErrorException ex)
{
Console.Error.WriteLineAsync(ex.ToString());
Environment.Exit(-1);
}
catch (TimeoutException)
{
Console.Error.WriteAsync(
$"Unable to obtain lock in {nameof(Soak)} method of {nameof(WashTask)} task.");
throw;
}
finally
{
sw?.Reset();
Debug.WriteLine("Ending soak.");
}
}
protected virtual void Cleanse(CancellationTokenPair token)
{
byte unitsToRemove;
{
using var lls = LaundryFlags.SpinLock(TimeSpan.FromSeconds(2));
unitsToRemove = lls.LoadedLaundryItem != LaundryItems.InvalidItem
? lls.LoadedLaundryItem.SoiledFactor
: throw new StateLogicErrorException(
"It should not be possible to lack laundry during the wash cycle.");
}
byte unitsToDeduct = unitsToRemove;
TimeSpan timeRequired = TimeToRemoveOneUnitOfSoil * unitsToRemove;
Stopwatch sw = null;
try
{
sw = HighPrecisionTimer;
sw.Restart();
if (timeRequired > TimeSpan.Zero)
{
SimulateWait(token, timeRequired, "Beginning cleanse wait.");
}
}
catch (IndividualOperationCancelledException)
{
if (sw == null)
{
string log = "For some reason the stopwatch is null.";
TerminationHelper.TerminateApplication(log);
return;
}
TimeSpan elapsed = sw.Elapsed;
double percentage = elapsed >= timeRequired ? 1.0 : elapsed / timeRequired;
unitsToDeduct = Convert.ToByte(Math.Floor(unitsToRemove * percentage));
throw;
}
finally
{
sw?.Reset();
byte newSetting = (byte)(unitsToRemove - unitsToDeduct);
Debug.Assert(newSetting <= unitsToRemove);
using var lls = LaundryFlags.SpinLock(TimeSpan.FromSeconds(2));
lls.SetSoilFactor(newSetting);
Debug.Assert(lls.LoadedLaundryItem.SoiledFactor <= unitsToRemove);
}
}
internal override async Task SendAsync(IOperation op, CancellationTokenPair tokenPair = default)
{
if (KeyMapper == null)
{
ThrowHelper.ThrowInvalidOperationException($"Bucket {Name} is not bootstrapped.");
}
if (op.RequiresVBucketId)
{
var vBucket = (VBucket)KeyMapper.MapKey(op.Key, op.WasNmvb());
var endPoint = op.ReplicaIdx != null && op.ReplicaIdx > -1
? vBucket.LocateReplica(op.ReplicaIdx.GetValueOrDefault())
: vBucket.LocatePrimary();
op.VBucketId = vBucket.Index;
try
{
if (Nodes.TryGet(endPoint.GetValueOrDefault(), out var clusterNode))
{
await clusterNode.SendAsync(op, tokenPair).ConfigureAwait(false);
return;
}
}
catch (ArgumentNullException)
{
//We could not find a candidate node to send so put into the retry queue
//its likely were between cluster map updates and we'll try again later
throw new NodeNotAvailableException(
$"Cannot find a Couchbase Server node for {endPoint}.");
}
}
//Make sure we use a node with the data service
var node = Nodes.GetRandom(x => x.HasKv);
if (node == null)
{
throw new NodeNotAvailableException(
$"Cannot find a Couchbase Server node for executing {op.GetType()}.");
}
await node.SendAsync(op, tokenPair).ConfigureAwait(false);
}
internal override Task SendAsync(IOperation op, CancellationTokenPair tokenPair = default)
{
if (KeyMapper == null)
{
throw new InvalidOperationException("Bucket is not bootstrapped.");
}
var bucket = KeyMapper.MapKey(op.Key);
var endPoint = bucket.LocatePrimary();
if (Nodes.TryGet(endPoint, out var clusterNode))
{
return(clusterNode.ExecuteOp(op, tokenPair));
}
//raise exception that node is not found
return(Task.CompletedTask);
}
protected async Task <T> SendAsync <T>(IOperation <T> op, IConnection connection, CancellationToken cancellationToken)
{
await op.SendAsync(connection, cancellationToken).ConfigureAwait(false);
ResponseStatus status;
using (new OperationCancellationRegistration(op, CancellationTokenPair.FromInternalToken(cancellationToken)))
{
status = await op.Completed.ConfigureAwait(false);
}
if (status != ResponseStatus.Success && status != ResponseStatus.AuthenticationContinue)
{
throw new AuthenticationFailureException(
$"Cannot authenticate the user. Reason: {status}");
}
return(op.GetValue() !);
}
public async Task ExternalCancellation_ReadOnlyOperation_Cancels(bool isSent)
{
// Arrange
var operation = new Get <dynamic>
{
IsSent = isSent
};
var cts = new CancellationTokenSource(100);
var tokenPair = CancellationTokenPair.FromExternalToken(cts.Token);
// Act
using var registration = new OperationCancellationRegistration(operation, tokenPair);
// Assert
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => operation.Completed.AsTask());
}
public async Task ExternalCancellation_MutationOperationNotSent_Cancels()
{
// Arrange
var operation = new Set <dynamic>("fake", "fakeKey")
{
IsSent = false
};
var cts = new CancellationTokenSource(100);
var tokenPair = CancellationTokenPair.FromExternalToken(cts.Token);
// Act
using var registration = new OperationCancellationRegistration(operation, tokenPair);
// Assert
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => operation.Completed.AsTask());
}
public void Dispose_CancelsRegistration()
{
// Arrange
var operation = new Get <dynamic>();
var cts = new CancellationTokenSource();
var tokenPair = CancellationTokenPair.FromInternalToken(cts.Token);
// Act
var registration = new OperationCancellationRegistration(operation, tokenPair);
registration.Dispose();
// Assert
cts.Cancel();
Assert.False(operation.Completed.IsCompleted);
}
public async Task Operation_Succeeds_Without_Retry()
{
var retryOrchestrator = CreateRetryOrchestrator();
var op = new Get <dynamic> {
RetryStrategy = new BestEffortRetryStrategy()
};
var bucketMock = new Mock <BucketBase>("fake", new ClusterContext(), new Mock <IScopeFactory>().Object,
retryOrchestrator, new Mock <ILogger>().Object, new Mock <IRedactor>().Object,
new Mock <IBootstrapperFactory>().Object, NoopRequestTracer.Instance,
new Mock <IOperationConfigurator>().Object,
new BestEffortRetryStrategy());
bucketMock.Setup(x => x.SendAsync(op, It.IsAny <CancellationTokenPair>()))
.Returns(Task.CompletedTask);
var tokenSource = new CancellationTokenSource(TimeSpan.FromMilliseconds(2500));
await retryOrchestrator.RetryAsync(bucketMock.Object, op, CancellationTokenPair.FromInternalToken(tokenSource.Token)).ConfigureAwait(false);
Assert.Equal(1u, op.Attempts);
}
public void IsCancellationRequested_TokensCanceled_True(bool cancelInternal, bool cancelExternal)
{
// Arrange
using var cts = new CancellationTokenSource();
using var cts2 = new CancellationTokenSource();
var tokenPair = new CancellationTokenPair(cts.Token, cts2.Token);
if (cancelExternal)
{
cts.Cancel();
}
if (cancelInternal)
{
cts2.Cancel();
}
// Assert
Assert.True(tokenPair.IsCancellationRequested);
}
internal override async Task SendAsync(IOperation op, CancellationTokenPair tokenPair = default)
{
if (KeyMapper == null)
{
ThrowHelper.ThrowInvalidOperationException($"Bucket {Name} is not bootstrapped.");
}
if (op.RequiresVBucketId)
{
var vBucket = (VBucket)KeyMapper.MapKey(op.Key);
var endPoint = op.ReplicaIdx != null
? vBucket.LocateReplica(op.ReplicaIdx.GetValueOrDefault())
: vBucket.LocatePrimary();
op.VBucketId = vBucket.Index;
if (Nodes.TryGet(endPoint !, out var clusterNode))
{
await clusterNode.SendAsync(op, tokenPair);
return;
}
throw new NodeNotAvailableException(
$"Cannot find a Couchbase Server node for {endPoint}.");
}
var node = Nodes.GetRandom();
if (node == null)
{
throw new NodeNotAvailableException(
$"Cannot find a Couchbase Server node for executing {op.GetType()}.");
}
await node.SendAsync(op, tokenPair);
}
private async Task AssertDoesNotRetryAsync(IOperation op, Exception exp)
{
var retryOrchestrator = CreateRetryOrchestrator();
var bucketMock = new Mock <BucketBase>("name", new ClusterContext(), new Mock <IScopeFactory>().Object,
retryOrchestrator, new Mock <ILogger>().Object, new Mock <IRedactor>().Object,
new Mock <IBootstrapperFactory>().Object,
NoopRequestTracer.Instance,
new Mock <IOperationConfigurator>().Object,
new BestEffortRetryStrategy())
{
CallBase = true
};
bucketMock.Setup(x => x.SendAsync(op, It.IsAny <CancellationTokenPair>())).Throws(exp);
var tokenSource = new CancellationTokenSource(TimeSpan.FromMilliseconds(2500));
try
{
await bucketMock.Object.RetryAsync(op, CancellationTokenPair.FromInternalToken(tokenSource.Token)).ConfigureAwait(false);
}
catch (Exception e)
{
if (e.GetType() == exp.GetType())
{
//expected
}
else
{
throw;
}
}
Assert.True(op.Attempts == 1);
}
public async Task ExternalCancellation_MutationOperationSent_WaitsForInternalCancellation()
{
// Arrange
var operation = new Set <dynamic>("fake", "fakeKey")
{
IsSent = true
};
var externalCts = new CancellationTokenSource(100);
var internalCts = new CancellationTokenSource();
var tokenPair = new CancellationTokenPair(externalCts.Token, internalCts.Token);
// Act
using var registration = new OperationCancellationRegistration(operation, tokenPair);
tokenPair.Register(() => internalCts.CancelAfter(100));
// Assert
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => operation.Completed.AsTask());
Assert.True(internalCts.IsCancellationRequested);
}
protected override TaskResult ExecuteTask(CancellationTokenPair pair)
{
bool needToClearError;
using (var lsfLck = LaundryFlags.Lock(TimeSpan.FromSeconds(2)))
{
pair.ThrowIfCancellationRequested();
needToClearError = lsfLck.ExecuteQuery((in LaundryStatusFlags lsf) =>
lsf.ErrorRegistrationStatus != ErrorRegistrationStatus.NilStatus);
}
if (needToClearError)
{
using (var lsfLck = LaundryFlags.Lock(TimeSpan.FromSeconds(2)))
{
lsfLck.ExecuteAction((ref LaundryStatusFlags lsf) =>
{
bool processing = lsf.ProcessError();
if (!processing)
{
throw new StateLogicErrorException(
"Bad state ... error not being handled in correct sequence.");
}
});
}
ClearError(pair);
pair.ThrowIfCancellationRequested();
using (var lsfLck = LaundryFlags.Lock(TimeSpan.FromSeconds(2)))
{
lsfLck.ExecuteAction((ref LaundryStatusFlags lsf) =>
{
bool cleared = lsf.ClearError();
if (cleared)
{
lsf.ResetError();
}
else
{
throw new StateLogicErrorException("Ut oh, we couln't fix the error for some reason.");
}
});
}
pair.ThrowIfCancellationRequested();
}
TimeSpan simulatedTurnOnCycleTime;
try
{
using var rgen = RandomNumberSource.RGenVault.SpinLock();
simulatedTurnOnCycleTime = TimeSpan.FromSeconds(rgen.Value.Next(1, 4));
}
catch (TimeoutException ex)
{
Console.Error.WriteLineAsync($"Error getting lock on the rgen vault ... exception: [{ex}]");
simulatedTurnOnCycleTime = TimeSpan.FromSeconds(2);
}
SimulateWait(in pair, simulatedTurnOnCycleTime);
TaskResult ret;
using var lck = TaskResult.SpinLock(TimeSpan.FromSeconds(2));
ret = lck.Value = lck.Value.WithTerminationTaskResultType(TaskResultCode.SuccessResult);
return(ret);
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
operation.Token = tokenPair;
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToString(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode
});
}
}
internal override Task SendAsync(IOperation op, CancellationTokenPair token = default)
{
throw new NotImplementedException();
}
private static async Task <ConcurrentDictionary <string, IEnumerable <IEndpointDiagnostics> > > GetEndpointDiagnosticsAsync(ClusterContext context,
IEnumerable <IClusterNode> clusterNodes, bool ping, ICollection <ServiceType> serviceTypes, CancellationToken token)
{
var endpoints = new ConcurrentDictionary <string, IEnumerable <IEndpointDiagnostics> >();
IOperationConfigurator operationConfigurator = ping
? context.ServiceProvider.GetRequiredService <IOperationConfigurator>()
: null;
foreach (var clusterNode in clusterNodes)
{
if (serviceTypes.Contains(ServiceType.KeyValue) && clusterNode.HasKv)
{
var kvEndpoints = (List <IEndpointDiagnostics>)endpoints.GetOrAdd("kv", new List <IEndpointDiagnostics>());
foreach (var connection in clusterNode.ConnectionPool.GetConnections())
{
var endPointDiagnostics =
CreateEndpointHealth(clusterNode.Owner?.Name, DateTime.UtcNow, connection);
if (ping)
{
await RecordLatencyAsync(endPointDiagnostics, async() =>
{
var op = new Noop();
operationConfigurator.Configure(op);
await clusterNode.ExecuteOp(connection, op, CancellationTokenPair.FromExternalToken(token)).ConfigureAwait(false);
}).ConfigureAwait(false);
}
kvEndpoints.Add(endPointDiagnostics);
}
}
if (serviceTypes.Contains(ServiceType.Views) && clusterNode.HasViews)
{
if (clusterNode.Owner is CouchbaseBucket bucket)
{
var kvEndpoints = (List <IEndpointDiagnostics>)endpoints.GetOrAdd("view", new List <IEndpointDiagnostics>());
var endPointDiagnostics = CreateEndpointHealth(bucket.Name, ServiceType.Views, DateTime.UtcNow, clusterNode.LastViewActivity, clusterNode.EndPoint);
if (ping)
{
await RecordLatencyAsync(endPointDiagnostics,
async() => await bucket.ViewQueryAsync <object, object>("p", "p").ConfigureAwait(false))
.ConfigureAwait(false);
}
kvEndpoints.Add(endPointDiagnostics);
}
}
if (serviceTypes.Contains(ServiceType.Query) && clusterNode.HasQuery)
{
var kvEndpoints = (List <IEndpointDiagnostics>)endpoints.GetOrAdd("n1ql", new List <IEndpointDiagnostics>());
var endPointDiagnostics = CreateEndpointHealth("Cluster", ServiceType.Query, DateTime.UtcNow, clusterNode.LastQueryActivity, clusterNode.EndPoint);
if (ping)
{
await RecordLatencyAsync(endPointDiagnostics,
() => context.Cluster.QueryAsync <dynamic>("SELECT 1;"))
.ConfigureAwait(false);
}
kvEndpoints.Add(endPointDiagnostics);
}
if (serviceTypes.Contains(ServiceType.Analytics) && clusterNode.HasAnalytics)
{
var kvEndpoints = (List <IEndpointDiagnostics>)endpoints.GetOrAdd("cbas", new List <IEndpointDiagnostics>());
var endPointDiagnostics = CreateEndpointHealth("Cluster", ServiceType.Analytics, DateTime.UtcNow, clusterNode.LastQueryActivity, clusterNode.EndPoint);
if (ping)
{
await RecordLatencyAsync(endPointDiagnostics,
() => context.Cluster.AnalyticsQueryAsync <dynamic>("SELECT 1;"))
.ConfigureAwait(false);
}
kvEndpoints.Add(endPointDiagnostics);
}
if (serviceTypes.Contains(ServiceType.Search) && clusterNode.HasSearch)
{
var kvEndpoints = (List <IEndpointDiagnostics>)endpoints.GetOrAdd("fts", new List <IEndpointDiagnostics>());
var endPointDiagnostics = CreateEndpointHealth("Cluster", ServiceType.Search, DateTime.UtcNow, clusterNode.LastQueryActivity, clusterNode.EndPoint);
if (ping)
{
var index = "ping";
await RecordLatencyAsync(endPointDiagnostics,
() => context.Cluster.SearchQueryAsync(index, new NoOpQuery())).ConfigureAwait(false);
}
kvEndpoints.Add(endPointDiagnostics);
}
}
return(endpoints);
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
if (operation.Attempts > 1)
{
MetricTracker.KeyValue.TrackRetry(operation.OpCode);
}
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(operation is not GetCid &&
(e is ScopeNotFoundException ||
e is CollectionNotFoundException))
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
LogRefreshingCollectionId(e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
operation.IncrementAttempts(reason);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
operation.IncrementAttempts(reason);
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
MetricTracker.KeyValue.TrackTimeout(operation.OpCode);
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToStringInvariant(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode,
DispatchedFrom = operation.LastDispatchedFrom,
DispatchedTo = operation.LastDispatchedTo,
RetryReasons = operation.RetryReasons
});
}
}
