internal IndexBucketModel(BucketBase entity)
{
this.BucketId = entity.BucketId;
this.ModifyDate = entity.ModifyDate ?? entity.CreationDate;
this.CreatedBy = entity.CreatedBy;
this.BucketName = entity.BucketName;
}
public IEnumerable <IScope> CreateScopes(BucketBase bucket, Manifest manifest)
{
if (bucket == null)
{
throw new ArgumentNullException(nameof(bucket));
}
if (manifest == null)
{
throw new ArgumentNullException(nameof(manifest));
}
foreach (var scopeDef in manifest.scopes)
{
var collections = new List <ICollection>();
foreach (var collectionDef in scopeDef.collections)
{
collections.Add(_collectionFactory.Create(bucket,
Convert.ToUInt32(collectionDef.uid, 16),
collectionDef.name,
scopeDef.name));
}
yield return(new Scope(scopeDef.name, scopeDef.uid, collections, bucket, _scopeLogger));
}
}
internal CouchbaseCollection(BucketBase bucket, ClusterContext context, uint?cid, string name)
{
Cid = cid;
Name = name;
_bucket = bucket;
_context = context;
}
public Scope(ScopeDef?scopeDef, ICollectionFactory collectionFactory, BucketBase bucket, ILogger <Scope> logger)
{
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (scopeDef != null)
{
Name = scopeDef.name;
Id = scopeDef.uid;
_collections = new ConcurrentDictionary <string, ICouchbaseCollection>(
scopeDef.collections
.Select(p => collectionFactory.Create(bucket, this, Convert.ToUInt32(p.uid, 16), p.name))
.ToDictionary(x => x.Name, v => v));
}
else
{
Name = DefaultScopeName;
Id = DefaultScopeId;
_collections = new ConcurrentDictionary <string, ICouchbaseCollection>();
_collections.TryAdd(CouchbaseCollection.DefaultCollectionName,
collectionFactory.Create(bucket, this, null, CouchbaseCollection.DefaultCollectionName));
}
_queryContext = $"{_bucket.Name}.{Name}";
}
// protected internal for a unit test seam
protected internal virtual async Task <bool> RefreshCollectionId(BucketBase bucket, IOperation op)
{
try
{
var scope = await bucket.ScopeAsync(op.SName !).ConfigureAwait(false);
if (await scope.CollectionAsync(op.CName !).ConfigureAwait(false) is IInternalCollection collection)
{
//re-fetch the CID but do not allow retries in that path and force the CID update
await collection.PopulateCidAsync(false, true);
op.Reset();
op.Cid = collection.Cid;
return(true);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting new collection id.");
}
op.Reset();
return(false);
}
// protected internal for a unit test seam
protected internal virtual async Task <bool> RefreshCollectionId(BucketBase bucket, IOperation op)
{
try
{
var vBucket = (VBucket)bucket.KeyMapper !.MapKey(op.Key);
var endPoint = op.ReplicaIdx > 0 ? vBucket.LocateReplica(op.ReplicaIdx) : vBucket.LocatePrimary();
if (bucket.Nodes.TryGet(endPoint !, out var clusterNode))
{
var scope = bucket.Scope(op.SName);
var collection = (CouchbaseCollection)scope.Collection(op.CName);
var newCid = await clusterNode.GetCid($"{op.SName}.{op.CName}").ConfigureAwait(false);
collection.Cid = newCid;
op.Reset();
op.Cid = newCid;
return(true);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting new collection id.");
}
op.Reset();
return(false);
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default,
TimeSpan?timeout = null)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
await bucket.SendAsync(operation, token, timeout).ConfigureAwait(false);
break;
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.DurationValue.HasValue)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason}.", operation.Opaque,
operation.Key, reason);
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
break; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (TaskCanceledException) { ThrowTimeoutException(operation, timeout); }
catch (OperationCanceledException) { ThrowTimeoutException(operation, timeout); }
}
public Scope(string name, string id, IEnumerable <ICouchbaseCollection> collections, BucketBase bucket, ILogger <Scope> logger)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Id = id ?? throw new ArgumentNullException(nameof(id));
_collections = new ConcurrentDictionary <string, ICouchbaseCollection>(collections.ToDictionary(x => x.Name, v => v));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public IScope CreateDefaultScope(BucketBase bucket)
{
if (bucket == null)
{
throw new ArgumentNullException(nameof(bucket));
}
return(new Scope(null, _collectionFactory, bucket, _scopeLogger));
}
public CouchbaseCollection(BucketBase bucket, ITypeTranscoder transcoder, ILogger <CouchbaseCollection> logger,
uint?cid, string name, string scopeName)
{
Cid = cid;
Name = name ?? throw new ArgumentNullException(nameof(name));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
_transcoder = transcoder ?? throw new ArgumentNullException(nameof(transcoder));
ScopeName = scopeName;
}
public Scope(string name, BucketBase bucket, ICollectionFactory collectionFactory, ILogger <Scope> logger)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_collectionFactory = collectionFactory ?? throw new ArgumentNullException(nameof(collectionFactory));
_collections = new ConcurrentDictionary <string, ICouchbaseCollection>();
_queryContext = $"{_bucket.Name}.{Name}";
}
public Scope(string name, BucketBase bucket, ICollectionFactory collectionFactory, ILogger <Scope> logger)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_collectionFactory = collectionFactory ?? throw new ArgumentNullException(nameof(collectionFactory));
_collections = new ConcurrentDictionary <string, ICouchbaseCollection>();
_queryContext = $"default:{_bucket.Name.EscapeIfRequired()}.{name.EscapeIfRequired()}";
IsDefaultScope = name == DefaultScopeName;
}
public void Unsubscribe(BucketBase bucket)
{
ConfigChanged -= bucket.ConfigUpdated;
if (bucket is MemcachedBucket)
{
if (_httpConfigListeners.TryRemove(bucket.Name, out HttpStreamingConfigListener listener))
{
listener.Dispose();
}
}
}
public void Subscribe(BucketBase bucket)
{
ConfigChanged += bucket.ConfigUpdated;
if (bucket is MemcachedBucket)
{
var httpListener = new HttpStreamingConfigListener(bucket.Name, _configuration, _httpClient, this, TokenSource.Token);
if (_httpConfigListeners.TryAdd(bucket.Name, httpListener))
{
httpListener.StartListening();
}
}
}
internal CouchbaseCollection(BucketBase bucket, ITypeTranscoder transcoder, ILogger <CouchbaseCollection> logger,
ILogger <GetResult> getLogger, IRedactor redactor,
uint?cid, string name, IScope scope)
{
Cid = cid;
Name = name ?? throw new ArgumentNullException(nameof(name));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
Redactor = redactor ?? throw new ArgumentNullException(nameof(redactor));
_transcoder = transcoder ?? throw new ArgumentNullException(nameof(transcoder));
_getLogger = getLogger ?? throw new ArgumentNullException(nameof(getLogger));
Scope = scope ?? throw new ArgumentNullException(nameof(scope));
}
internal CouchbaseCollection(BucketBase bucket, IOperationConfigurator operationConfigurator, ILogger <CouchbaseCollection> logger,
ILogger <GetResult> getLogger, IRedactor redactor,
uint?cid, string name, IScope scope, IRequestTracer tracer)
{
Cid = cid;
Name = name ?? throw new ArgumentNullException(nameof(name));
_bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
Redactor = redactor ?? throw new ArgumentNullException(nameof(redactor));
_operationConfigurator = operationConfigurator ?? throw new ArgumentNullException(nameof(operationConfigurator));
_getLogger = getLogger ?? throw new ArgumentNullException(nameof(getLogger));
_tracer = tracer;
Scope = scope ?? throw new ArgumentNullException(nameof(scope));
}
public IScope CreateDefaultScope(BucketBase bucket)
{
if (bucket == null)
{
throw new ArgumentNullException(nameof(bucket));
}
var collections = new List <ICollection>
{
_collectionFactory.Create(bucket, null,
CouchbaseCollection.DefaultCollectionName, Scope.DefaultScopeName)
};
return(new Scope(Scope.DefaultScopeName, "0", collections, bucket, _scopeLogger));
}
public void Unsubscribe(BucketBase bucket)
{
lock (_configChangedSubscribers)
{
_configChangedSubscribers.Remove(bucket);
}
if (bucket is MemcachedBucket)
{
if (_httpConfigListeners.TryRemove(bucket.Name, out var listener))
{
listener.Dispose();
}
}
}
public IEnumerable <IScope> CreateScopes(BucketBase bucket, Manifest manifest)
{
if (bucket == null)
{
throw new ArgumentNullException(nameof(bucket));
}
if (manifest == null)
{
throw new ArgumentNullException(nameof(manifest));
}
foreach (var scopeDef in manifest.scopes)
{
yield return(new Scope(scopeDef, _collectionFactory, bucket, _scopeLogger));
}
}
public void Subscribe(BucketBase bucket)
{
ConfigChanged += bucket.ConfigUpdated;
if (bucket is MemcachedBucket)
{
var httpListener = _configListenerFactory.Create(bucket.Name, this);
if (_httpConfigListeners.TryAdd(bucket.Name, httpListener))
{
httpListener.StartListening();
// Dispose the listener when we're stopped
TokenSource.Token.Register(state =>
{
((HttpStreamingConfigListener)state).Dispose();
}, httpListener);
}
}
}
// protected internal for a unit test seam
protected internal virtual async Task <bool> RefreshCollectionId(BucketBase bucket, IOperation op)
{
try
{
var scope = await bucket.ScopeAsync(op.SName).ConfigureAwait(false);
var collection = (IInternalCollection)await scope.CollectionAsync(op.CName).ConfigureAwait(false);
var newCid = await((IInternalScope)scope).GetCidAsync($"{op.SName}.{op.CName}").ConfigureAwait(false);
collection.Cid = newCid;
op.Reset();
op.Cid = newCid;
return(true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting new collection id.");
}
op.Reset();
return(false);
}
public ICollection Create(BucketBase bucket, uint?cid, string name, string scopeName) => new CouchbaseCollection(bucket, _transcoder, _logger, cid, name, scopeName);
internal CouchbaseCollection(BucketBase bucket, uint?cid, string name)
{
Cid = cid;
Name = name;
_bucket = bucket;
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationToken token = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, token).ConfigureAwait(false);
break;
}
catch (CollectionOutdatedException e)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e)
{
if (e is IRetryable)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
token.ThrowIfCancellationRequested();
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
operation.Reset();
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
operation.Reset();
await Task.Delay(action.DurationValue.Value, token).ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
else
{
throw;
}
}
} while (true);
}
catch (OperationCanceledException) { ThrowTimeoutException(operation, operation.Timeout); }
}
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
operation.Token = tokenPair;
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
operation.Attempts++;
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(e is ScopeNotFoundException || e is CollectionNotFoundException)
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
_logger.LogInformation("Updating stale manifest for collection and retrying.", e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and always retry.",
operation.Opaque,
operation.Key, reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
_logger.LogDebug("Retrying op {opaque}/{key} because {reason} and action duration.",
operation.Opaque,
operation.Key, reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToString(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode
});
}
}
public ICouchbaseCollection Create(BucketBase bucket, IScope scope, string name) => new CouchbaseCollection(bucket, _operationConfigurator, _logger, _getLogger, _redactor, name, scope, _tracer);
public ICouchbaseCollection Create(BucketBase bucket, IScope scope, uint?cid, string name) => new CouchbaseCollection(bucket, _transcoder, _logger, _getLogger, _redactor, cid, name, scope);
public async Task RetryAsync(BucketBase bucket, IOperation operation, CancellationTokenPair tokenPair = default)
{
try
{
var backoff = ControlledBackoff.Create();
do
{
tokenPair.ThrowIfCancellationRequested();
try
{
if (operation.Attempts > 1)
{
MetricTracker.KeyValue.TrackRetry(operation.OpCode);
}
try
{
await bucket.SendAsync(operation, tokenPair).ConfigureAwait(false);
break;
}
catch (CouchbaseException e) when(operation is not GetCid &&
(e is ScopeNotFoundException ||
e is CollectionNotFoundException))
{
// We catch CollectionOutdatedException separately from the CouchbaseException catch block
// in case RefreshCollectionId fails. This causes that failure to trigger normal retry logic.
LogRefreshingCollectionId(e);
if (!await RefreshCollectionId(bucket, operation)
.ConfigureAwait(false))
{
// rethrow if we fail to refresh he collection ID so we hit retry logic
// otherwise we'll loop and retry immediately
throw;
}
}
}
catch (CouchbaseException e) when(e is IRetryable && !tokenPair.IsCancellationRequested)
{
var reason = e.ResolveRetryReason();
if (reason.AlwaysRetry())
{
LogRetryDueToAlwaysRetry(operation.Opaque, _redactor.UserData(operation.Key), reason);
await backoff.Delay(operation).ConfigureAwait(false);
// no need to reset op in this case as it was not actually sent
if (reason != RetryReason.CircuitBreakerOpen)
{
operation.Reset();
}
operation.IncrementAttempts(reason);
continue;
}
var strategy = operation.RetryStrategy;
var action = strategy.RetryAfter(operation, reason);
if (action.Retry)
{
LogRetryDueToDuration(operation.Opaque, _redactor.UserData(operation.Key), reason);
// Reset first so operation is not marked as sent if canceled during the delay
operation.Reset();
operation.IncrementAttempts(reason);
await Task.Delay(action.DurationValue.GetValueOrDefault(), tokenPair)
.ConfigureAwait(false);
}
else
{
throw; //don't retry
}
}
} while (true);
}
catch (OperationCanceledException) when(!tokenPair.IsExternalCancellation)
{
MetricTracker.KeyValue.TrackTimeout(operation.OpCode);
ThrowHelper.ThrowTimeoutException(operation, new KeyValueErrorContext
{
BucketName = operation.BucketName,
ClientContextId = operation.Opaque.ToStringInvariant(),
DocumentKey = operation.Key,
Cas = operation.Cas,
CollectionName = operation.CName,
ScopeName = operation.SName,
OpCode = operation.OpCode,
DispatchedFrom = operation.LastDispatchedFrom,
DispatchedTo = operation.LastDispatchedTo,
RetryReasons = operation.RetryReasons
});
}
}
public IScope CreateScope(string name, BucketBase bucket)
{
return(new Scope(name, bucket, _collectionFactory, _scopeLogger));
}
