/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="token">The <see cref="MutationToken"/> to compare against.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <param name="cts"></param>
/// <returns> A <see cref="Task{boolean}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task <bool> ObserveAsync(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo, CancellationTokenSource cts)
{
var keyMapper = (VBucketKeyMapper)_configInfo.GetKeyMapper();
var obParams = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
obParams.CheckConfiguredReplicas();
var persisted = await CheckPersistToAsync(obParams).ContinueOnAnyContext();
var replicated = await CheckReplicasAsync(obParams).ContinueOnAnyContext();
if (persisted && replicated)
{
Log.Debug("Persisted and replicated on first try: {0}", _key);
return(true);
}
return(await ObserveEveryAsync(async p =>
{
Log.Debug("trying again: {0}", _key);
persisted = await CheckPersistToAsync(obParams).ContinueOnAnyContext();
replicated = await CheckReplicasAsync(obParams).ContinueOnAnyContext();
return persisted & replicated;
}, obParams, _interval, cts.Token));
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="observeParams">The <see cref="ObserveParams"/> object.</param>
/// <param name="operation">The Observe operation.</param>
/// <param name="replicaIndex">The replicaIndex of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task <bool> CheckReplicaAsync(ObserveParams observeParams, Observe operation, int replicaIndex)
{
Log.Debug(m => m("checking replica {0}", replicaIndex));
if (observeParams.IsDurabilityMet())
{
return(true);
}
//clone the operation since we already checked the primary and we want to maintain internal state (opaque, timer, etc)
operation = (Observe)operation.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
var result = await Task.Run(() => replica.Send(operation)).ContinueOnAnyContext();
Log.Debug(m => m("Replica {0} - {1} [0]", replica.EndPoint, result.Value, replicaIndex));
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return(!observeParams.HasMutated(state.Cas));
}
public async Task <bool> ObserveAsync(string key, ulong cas, bool deletion, ReplicateTo replicateTo,
PersistTo persistTo, CancellationTokenSource cts)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
observeParams.CheckConfiguredReplicas();
var persisted = await CheckPersistToAsync(observeParams).ContinueOnAnyContext();
var replicated = await CheckReplicasAsync(observeParams).ContinueOnAnyContext();
if (persisted && replicated)
{
return(true);
}
return(await ObserveEveryAsync(async p =>
{
Log.Debug("trying again: {0}", key);
persisted = await CheckPersistToAsync(observeParams).ContinueOnAnyContext();
replicated = await CheckReplicasAsync(observeParams).ContinueOnAnyContext();
return persisted & replicated;
}, observeParams, _interval, cts.Token));
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="observeParams">The <see cref="ObserveParams"/> object.</param>
/// <param name="replicaIndex">The replicaIndex of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task <bool> CheckReplicaAsync(ObserveParams observeParams, int replicaIndex)
{
if (observeParams.IsDurabilityMet())
{
return(true);
}
var operation = new Observe(observeParams.Key, observeParams.VBucket, _clusterController.Transcoder, (uint)_timeout);
var tcs = new TaskCompletionSource <IOperationResult <ObserveState> >();
operation.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(operation.Opaque, operation);
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
await replica.SendAsync(operation).ContinueOnAnyContext();
var result = await tcs.Task.ContinueOnAnyContext();
Log.Debug("Replica {0} - {1} {2} - opaque: {3} key:{4}", replica.EndPoint, result.Value.KeyState, replicaIndex, operation.Opaque, observeParams.Key);
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return(!observeParams.HasMutated(state.Cas));
}
private async Task <bool> CheckPersistToAsync(ObserveParams observeParams)
{
if (observeParams.PersistTo == PersistTo.Zero)
{
return(true);
}
var op = new ObserveSeqno(observeParams.Token, _clusterController.Transcoder, _timeout);
observeParams.Operation = op;
var tcs = new TaskCompletionSource <IOperationResult <ObserveSeqnoResponse> >();
op.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(op.Opaque, op);
var server = await GetServerAsync(observeParams);
await server.SendAsync(op).ContinueOnAnyContext();
var response = await tcs.Task.ContinueOnAnyContext();
observeParams.CheckMutationLost(response);
observeParams.CheckPersisted(response);
return(observeParams.IsDurabilityMet());
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="op">The <see cref="ObserveParams"/> object.</param>
/// <param name="replicaIndex">The index of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task <bool> CheckReplica(ObserveParams op, int replicaIndex)
{
Log.Debug(m => m("checking replica {0}", replicaIndex));
if (op.IsDurabilityMet())
{
return(true);
}
var replica = op.VBucket.LocateReplica(replicaIndex);
var result = await Task.Run(() => replica.Send(new Observe(op.Key, op.VBucket, new AutoByteConverter())))
.ConfigureAwait(false);
Log.Debug(m => m("Replica {0} - {1} [0]", replica.EndPoint, result.Value, replicaIndex));
var state = result.Value;
if (state.KeyState == op.Criteria.PersistState)
{
Interlocked.Increment(ref op.ReplicatedToCount);
Interlocked.Increment(ref op.PersistedToCount);
}
else if (state.KeyState == op.Criteria.ReplicateState)
{
Interlocked.Increment(ref op.ReplicatedToCount);
}
return(op.IsDurabilityMet() && !op.HasMutated(state.Cas));
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns></returns>
public bool Observe(string key, ulong cas, bool deletion, ReplicateTo replicateTo, PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
//Used to terminate the loop at the specific timeout
using (var cancellationTokenSource = new CancellationTokenSource(_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = ObserveEvery(p =>
{
//check the master for persistence to disk
var master = p.VBucket.LocatePrimary();
var result = master.Send(new Observe(key, vBucket, new AutoByteConverter()));
Log.Debug(m => m("Master {0} - {1}", master.EndPoint, result.Value));
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
//Key mutation detected so fail
if (p.HasMutated(state.Cas))
{
return(false);
}
//Check if durability requirements have been met
if (p.IsDurabilityMet())
{
return(true);
}
//Run the durability requirement check on each replica
var tasks = new List <Task <bool> >();
var replicas = GetReplicas(vBucket, replicateTo, persistTo);
replicas.ForEach(x => tasks.Add(CheckReplica(p, x)));
//Wait for all tasks to finish
Task.WaitAll(tasks.ToArray());
return(tasks.All(subtask => subtask.Result));
}, observeParams, _interval, cancellationTokenSource.Token);
task.ConfigureAwait(false);
task.Wait(_timeout);
return(task.Result);
}
}
/// <summary>
/// Checks the replicas to see if the key has met the durability constraints defined by the caller.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="operation">The operation observe operation reference; will be cloned if reused.</param>
/// <returns></returns>
private bool CheckReplicas(ObserveParams observeParams, Observe operation)
{
//Get the candidate replicas, if none are defined that match the specified durability return false.
var replicas = observeParams.GetReplicas();
//Check each replica to see if has met the durability constraints specified. A mutation means we failed.
var notMutated = replicas.All(index => CheckReplica(observeParams, operation, index));
return(observeParams.IsDurabilityMet() && notMutated);
}
/// <summary>
/// Checks a replica for durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="op">The op.</param>
/// <param name="replicaId">The replica identifier.</param>
/// <returns></returns>
private bool CheckReplica(ObserveParams observeParams, ObserveSeqno op, int replicaId)
{
var cloned = (ObserveSeqno)op.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaId);
var result = replica.Send(cloned);
observeParams.CheckMutationLost(result);
observeParams.CheckPersisted(result);
observeParams.CheckReplicated(result);
return(observeParams.IsDurabilityMet());
}
private async Task <bool> CheckReplicasAsync(ObserveParams observeParams)
{
if (observeParams.ReplicateTo == ReplicateTo.Zero)
{
return(true);
}
var replicas = observeParams.GetReplicas().Select(x => CheckReplicaAsync(observeParams, x));
await Task.WhenAll(replicas);
return(observeParams.IsDurabilityMet());
}
async Task <bool> CheckReplicasAsync(ObserveParams observeParams, ObserveSeqno op)
{
var replicas = observeParams.GetReplicas();
var tasks = new List <Task <bool> >();
replicas.ForEach(x => tasks.Add(CheckReplicaAsync(observeParams, op, x)));
await Task.WhenAll(tasks);
return(observeParams.IsDurabilityMet());
}
static Task <bool> CheckReplicaAsync(ObserveParams observeParams, ObserveSeqno op, int replicaId)
{
var cloned = (ObserveSeqno)op.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaId);
var result = replica.Send(cloned);
observeParams.CheckMutationLost(result.Value);
observeParams.CheckPersisted(result.Value);
observeParams.CheckReplicated(result.Value);
return(Task.FromResult(observeParams.IsDurabilityMet()));
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key.
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns>True if the durability constraints have been met.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public bool Observe(string key, ulong cas, bool deletion, ReplicateTo replicateTo, PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var p = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
p.CheckConfiguredReplicas();
var operation = new Observe(key, vBucket, _clusterController.Transcoder, (uint)_timeout);
do
{
var master = p.VBucket.LocatePrimary();
var result = master.Send(operation);
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
if (!deletion && p.HasMutated(state.Cas))
{
return(false);
}
//First check if durability has already been met
if (p.IsDurabilityMet())
{
return(true);
}
//If not check each replica
if (CheckReplicas(p, operation))
{
return(true);
}
//prepare for another attempt
operation = (Observe)operation.Clone();
p.Reset();
} while (!operation.TimedOut());
return(false);
}
private async Task <IServer> GetServerAsync(ObserveParams p)
{
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10)
{
throw new TimeoutException("Could not acquire a server.");
}
await Task.Delay((int)Math.Pow(2, attempts)).ContinueOnAnyContext();
}
return(master);
}
/// <summary>
/// Checks the replicas to see if the key has met the durability constraints defined by the caller.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="operation">The operation observe operation reference; will be cloned if reused.</param>
/// <param name="replicateTo">The replication durability that must be met.</param>
/// <param name="persistTo">The persistence durbaility that must be met.</param>
/// <returns></returns>
bool CheckReplicas(ObserveParams observeParams, Observe operation, ReplicateTo replicateTo, PersistTo persistTo)
{
//Get the candidate replicas, if none are defined that match the specified durability return false.
var replicas = GetReplicas(observeParams.VBucket, replicateTo, persistTo);
if (replicas.Count < (int)replicateTo)
{
return(false);
}
//Check each replica to see if has met the durability constraints specified. A mutation means we failed.
var mutated = replicas.All(index => CheckReplica(observeParams, operation, index));
return(observeParams.IsDurabilityMet() && !mutated);
}
private async Task <bool> CheckReplicasAsync(ObserveParams observeParams)
{
if (observeParams.ReplicateTo == ReplicateTo.Zero)
{
return(true);
}
var replicas = observeParams.GetReplicas().Select(x => CheckReplicaAsync(observeParams, x)).ToList();
//Wait for all tasks to finish
await Task.WhenAll(replicas).ContinueOnAnyContext();
var notMutated = replicas.All(subtask => subtask.Result);
var durabilityMet = observeParams.IsDurabilityMet();
return(durabilityMet && notMutated);
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="token">The <see cref="MutationToken"/> to compare against.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns> A <see cref="Task{bool}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task <bool> ObserveAsync(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo)
{
var keyMapper = (VBucketKeyMapper)_configInfo.GetKeyMapper();
var obParams = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
obParams.CheckConfiguredReplicas();
var op = new ObserveSeqno(obParams.Token, _transcoder, _timeout);
using (var cts = new CancellationTokenSource((int)_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = await ObserveEvery(async p =>
{
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10)
{
throw new TimeoutException("Could not acquire a server.");
}
await Task.Delay((int)Math.Pow(2, attempts)).ContinueOnAnyContext();
}
var result = master.Send(op);
var osr = result.Value;
p.CheckMutationLost(osr);
p.CheckPersisted(osr);
if (p.IsDurabilityMet())
{
return(true);
}
return(await CheckReplicasAsync(p, op).ContinueOnAnyContext());
}, obParams, _interval, op, cts.Token).ContinueOnAnyContext();
return(task);
}
}
/// <summary>
/// Observes the specified key using the Seqno.
/// </summary>
/// <param name="key">The key.</param>
/// <param name="token">The token.</param>
/// <param name="replicateTo">The replicate to.</param>
/// <param name="persistTo">The persist to.</param>
/// <returns>True if durability constraints were matched.</returns>
/// <exception cref="DocumentMutationLostException">Thrown if the observed document was lost during
/// a hard failover because the document did not reach the replica in time.</exception>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public bool Observe(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo)
{
var keyMapper = (VBucketKeyMapper)_configInfo.GetKeyMapper();
var p = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
p.CheckConfiguredReplicas();
var op = new ObserveSeqno(p.Token, _transcoder, _timeout);
do
{
var master = p.VBucket.LocatePrimary();
var result = master.Send(op);
var osr = result.Value;
p.CheckMutationLost(osr);
p.CheckPersisted(osr);
if (p.IsDurabilityMet())
{
return(true);
}
if (CheckReplicas(p, op))
{
return(true);
}
//prepare for another attempt
op = (ObserveSeqno)op.Clone();
p.Reset();
} while (!op.TimedOut());
return(false);
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task <bool> ObserveEvery(Func <ObserveParams, bool> observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
if (observe(observeParams))
{
return(true);
}
var task = Task.Delay(interval, cancellationToken);
try
{
await task;
}
catch (TaskCanceledException)
{
return(false);
}
}
}
private async Task <bool> CheckPersistToAsync(ObserveParams observeParams)
{
if (observeParams.PersistTo == PersistTo.Zero)
{
return(true);
}
var tcs = new TaskCompletionSource <IOperationResult <ObserveState> >();
var operation = new Observe(observeParams.Key, observeParams.VBucket, _clusterController.Transcoder, (uint)_timeout);
operation.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(operation.Opaque, operation);
var server = await GetServerAsync(observeParams);
await server.SendAsync(operation).ContinueOnAnyContext();
var result = await tcs.Task.ContinueOnAnyContext();
Log.Debug("Master {0} - {1} key:{2}", server.EndPoint, result.Value, observeParams.Key);
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.PersistedToCount);
}
//Key mutation detected so fail
if (observeParams.HasMutated(state.Cas))
{
Log.Debug("Mutation detected {0} - {1} - opaque: {2} key:{3}", server.EndPoint, result.Value, operation.Opaque, observeParams.Key);
throw new DocumentMutationException(string.Format("Document mutation detected during observe for key '{0}'", observeParams.Key));
}
//Check if durability requirements have been met
if (observeParams.IsDurabilityMet())
{
Log.Debug("Durability met {0} - {1} - opaque: {2} key:{3}", server.EndPoint, result.Value, operation.Opaque, observeParams.Key);
return(true);
}
return(false);
}
/// <summary>
/// Checks the replica at a given replicaIndex for the durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters - stateful - gather info with each request.</param>
/// <param name="operation">The observe operation.</param>
/// <param name="replicaIndex">Index of the replica.</param>
/// <returns>True if the key has not mutated.</returns>
private static bool CheckReplica(ObserveParams observeParams, Observe operation, int replicaIndex)
{
//clone the operation since we already checked the primary and we want to maintain internal state (opaque, timer, etc)
operation = (Observe)operation.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
var result = replica.Send(operation);
//Check the result and update the counters
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return(!observeParams.HasMutated(state.Cas));
}
private async Task <bool> CheckReplicaAsync(ObserveParams observeParams, int replicaId)
{
var op = new ObserveSeqno(observeParams.Token, _clusterController.Transcoder, _timeout);
observeParams.Operation = op;
var tcs = new TaskCompletionSource <IOperationResult <ObserveSeqnoResponse> >();
op.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(op.Opaque, op);
Log.Debug("checking replica {0} - opaque: {1}", replicaId, op.Opaque);
var replica = observeParams.VBucket.LocateReplica(replicaId);
await replica.SendAsync(op).ContinueOnAnyContext();
var response = await tcs.Task.ContinueOnAnyContext();
observeParams.CheckMutationLost(response);
observeParams.CheckPersisted(response);
observeParams.CheckReplicated(response);
return(observeParams.IsDurabilityMet());
}
private async Task<bool> CheckReplicasAsync(ObserveParams observeParams)
{
if (observeParams.ReplicateTo == ReplicateTo.Zero) return true;
var replicas = observeParams.GetReplicas().Select(x=>CheckReplicaAsync(observeParams, x));
await Task.WhenAll(replicas);
return observeParams.IsDurabilityMet();
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task <bool> ObserveEvery(Func <ObserveParams, Task <bool> > observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return(true);
}
var task = Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
try
{
await task;
}
catch (TaskCanceledException)
{
return(false);
}
}
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns></returns>
public bool Observe(string key, ulong cas, bool deletion, ReplicateTo replicateTo, PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
//Used to terminate the loop at the specific timeout
using (var cancellationTokenSource = new CancellationTokenSource(_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = ObserveEvery(p =>
{
//check the master for persistence to disk
var master = p.VBucket.LocatePrimary();
var result = master.Send(new Observe(key, vBucket, new AutoByteConverter()));
Log.Debug(m => m("Master {0} - {1}", master.EndPoint, result.Value));
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
//Check if durability requirements have been met
if (p.IsDurabilityMet())
{
return true;
}
//Key mutation detected so fail
if (p.HasMutated(state.Cas))
{
return false;
}
//Run the durability requirement check on each replica
var tasks = new List<Task<bool>>();
var replicas = GetReplicas(vBucket, replicateTo, persistTo);
replicas.ForEach(x => tasks.Add(CheckReplica(p, x)));
//Wait for all tasks to finish
Task.WaitAll(tasks.ToArray());
return tasks.All(subtask => subtask.Result);
}, observeParams, _interval, cancellationTokenSource.Token);
task.Wait(_timeout);
return task.Result;
}
}
/// <summary>
/// Checks the replica at a given replicaIndex for the durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters - stateful - gather info with each request.</param>
/// <param name="operation">The observe operation.</param>
/// <param name="replicaIndex">Index of the replica.</param>
/// <returns>True if the key has not mutated.</returns>
static bool CheckReplica(ObserveParams observeParams, Observe operation, int replicaIndex)
{
//clone the operation since we already checked the primary and we want to maintain internal state (opaque, timer, etc)
operation = (Observe)operation.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
var result = replica.Send(operation);
//Check the result and update the counters
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return !observeParams.HasMutated(state.Cas);
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="token">The <see cref="MutationToken"/> to compare against.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns> A <see cref="Task{bool}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task<bool> ObserveAsync(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo)
{
var keyMapper = (VBucketKeyMapper)_configInfo.GetKeyMapper();
var obParams = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
obParams.CheckConfiguredReplicas();
var op = new ObserveSeqno(obParams.Token, _transcoder, _timeout);
using (var cts = new CancellationTokenSource((int)_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = await ObserveEvery(async p =>
{
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
await Task.Delay((int)Math.Pow(2, attempts)).ContinueOnAnyContext();
}
var result = master.Send(op);
var osr = result.Value;
p.CheckMutationLost(osr);
p.CheckPersisted(osr);
if (p.IsDurabilityMet())
{
return true;
}
return await CheckReplicasAsync(p, op).ContinueOnAnyContext();
}, obParams, _interval, op, cts.Token).ContinueOnAnyContext();
return task;
}
}
private async Task<bool> CheckReplicasAsync(ObserveParams observeParams)
{
if (observeParams.ReplicateTo == ReplicateTo.Zero) return true;
var replicas = observeParams.GetReplicas().Select(x=>CheckReplicaAsync(observeParams, x)).ToList();
//Wait for all tasks to finish
await Task.WhenAll(replicas).ContinueOnAnyContext();
var notMutated = replicas.All(subtask => subtask.Result);
var durabilityMet = observeParams.IsDurabilityMet();
return durabilityMet && notMutated;
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key.
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns>True if the durability constraints have been met.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public bool Observe(string key, ulong cas, bool deletion, ReplicateTo replicateTo, PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var p = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
p.CheckConfiguredReplicas();
var operation = new Observe(key, vBucket, _transcoder, (uint)_timeout);
do
{
var master = p.VBucket.LocatePrimary();
var result = master.Send(operation);
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
if (!deletion && p.HasMutated(state.Cas))
{
return false;
}
//First check if durability has already been met
if (p.IsDurabilityMet())
{
return true;
}
//If not check each replica
if (CheckReplicas(p, operation))
{
return true;
}
//prepare for another attempt
operation = (Observe)operation.Clone();
p.Reset();
} while (!operation.TimedOut());
return false;
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task<bool> ObserveEvery(Func<ObserveParams, Task<bool>> observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return true;
}
var task = Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
try
{
await task;
}
catch (TaskCanceledException)
{
return false;
}
}
}
private async Task<IServer> GetServerAsync(ObserveParams p)
{
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10)
{
throw new TimeoutException("Could not acquire a server.");
}
await Task.Delay((int)Math.Pow(2, attempts)).ContinueOnAnyContext();
}
return master;
}
async Task<bool> CheckReplicasAsync(ObserveParams observeParams, ObserveSeqno op)
{
var replicas = observeParams.GetReplicas();
var tasks = new List<Task<bool>>();
replicas.ForEach(x => tasks.Add(CheckReplicaAsync(observeParams, op, x)));
await Task.WhenAll(tasks);
return observeParams.IsDurabilityMet();
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="token">The <see cref="MutationToken"/> to compare against.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <param name="cts"></param>
/// <returns> A <see cref="Task{boolean}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task<bool> ObserveAsync(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo, CancellationTokenSource cts)
{
var keyMapper = (VBucketKeyMapper) _configInfo.GetKeyMapper();
var obParams = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
obParams.CheckConfiguredReplicas();
var persisted = await CheckPersistToAsync(obParams).ContinueOnAnyContext();
var replicated = await CheckReplicasAsync(obParams).ContinueOnAnyContext();
if (persisted && replicated)
{
Log.DebugFormat("Persisted and replicated on first try: {0}", _key);
return true;
}
return await ObserveEveryAsync(async p =>
{
Log.DebugFormat("trying again: {0}", _key);
persisted = await CheckPersistToAsync(obParams).ContinueOnAnyContext();
replicated = await CheckReplicasAsync(obParams).ContinueOnAnyContext();
return persisted & replicated;
}, obParams, _interval, cts.Token);
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="op">The <see cref="Observe"/> operation.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task<bool> ObserveEveryAsync(Func<ObserveParams, Task<bool>> observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return true;
}
await Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
}
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="observeParams">The <see cref="ObserveParams"/> object.</param>
/// <param name="replicaIndex">The replicaIndex of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task<bool> CheckReplicaAsync(ObserveParams observeParams, int replicaIndex)
{
if (observeParams.IsDurabilityMet()) return true;
var operation = new Observe(observeParams.Key, observeParams.VBucket, _clusterController.Transcoder, (uint)_timeout);
var tcs = new TaskCompletionSource<IOperationResult<ObserveState>>();
operation.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(operation.Opaque, operation);
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
await replica.SendAsync(operation).ContinueOnAnyContext();
var result = await tcs.Task.ContinueOnAnyContext();
Log.Debug(m=>m("Replica {0} - {1} {2} - opaque: {3} key:{4}", replica.EndPoint, result.Value.KeyState, replicaIndex, operation.Opaque, observeParams.Key));
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return !observeParams.HasMutated(state.Cas);
}
private async Task<bool> CheckPersistToAsync(ObserveParams observeParams)
{
if (observeParams.PersistTo == PersistTo.Zero) return true;
var tcs = new TaskCompletionSource<IOperationResult<ObserveState>>();
var operation = new Observe(observeParams.Key, observeParams.VBucket, _clusterController.Transcoder, (uint)_timeout);
operation.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(operation.Opaque, operation);
var server = await GetServerAsync(observeParams);
await server.SendAsync(operation).ContinueOnAnyContext();
var result = await tcs.Task.ContinueOnAnyContext();
Log.Debug(m => m("Master {0} - {1} key:{2}", server.EndPoint, result.Value, observeParams.Key));
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.PersistedToCount);
}
//Key mutation detected so fail
if (observeParams.HasMutated(state.Cas))
{
Log.Debug(m => m("Mutation detected {0} - {1} - opaque: {2} key:{3}",server.EndPoint, result.Value, operation.Opaque, observeParams.Key));
throw new DocumentMutationException(string.Format("Document mutation detected during observe for key '{0}'", observeParams.Key));
}
//Check if durability requirements have been met
if (observeParams.IsDurabilityMet())
{
Log.Debug(m => m("Durability met {0} - {1} - opaque: {2} key:{3}", server.EndPoint, result.Value, operation.Opaque, observeParams.Key));
return true;
}
return false;
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns> A <see cref="Task{bool}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task <bool> ObserveAsync(string key, ulong cas, bool deletion, ReplicateTo replicateTo,
PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
observeParams.CheckConfiguredReplicas();
var operation = new Observe(key, vBucket, _transcoder, (uint)_timeout);
//Used to terminate the loop at the specific timeout
using (var cts = new CancellationTokenSource(_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = await ObserveEvery(async p =>
{
//check the master for persistence to disk
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10)
{
throw new TimeoutException("Could not acquire a server.");
}
await Task.Delay((int)Math.Pow(2, attempts));
}
var result = master.Send(operation);
Log.Debug(m => m("Master {0} - {1}", master.EndPoint, result.Value));
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
//Key mutation detected so fail
if (p.HasMutated(state.Cas))
{
return(false);
}
//Check if durability requirements have been met
if (p.IsDurabilityMet())
{
return(true);
}
//Run the durability requirement check on each replica
var tasks = new List <Task <bool> >();
var replicas = p.GetReplicas();
replicas.ForEach(x => tasks.Add(CheckReplicaAsync(p, operation, x)));
//Wait for all tasks to finish
await Task.WhenAll(tasks.ToArray()).ContinueOnAnyContext();
var notMutated = tasks.All(subtask => subtask.Result);
return(p.IsDurabilityMet() && notMutated);
}, observeParams, operation, _interval, cts.Token).ContinueOnAnyContext();
return(task);
}
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="op">The <see cref="Observe"/> operation.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task <bool> ObserveEveryAsync(Func <ObserveParams, Task <bool> > observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return(true);
}
await Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
}
}
/// <summary>
/// Observes the specified key using the Seqno.
/// </summary>
/// <param name="key">The key.</param>
/// <param name="token">The token.</param>
/// <param name="replicateTo">The replicate to.</param>
/// <param name="persistTo">The persist to.</param>
/// <returns>True if durability constraints were matched.</returns>
/// <exception cref="DocumentMutationLostException">Thrown if the observed document was lost during
/// a hard failover because the document did not reach the replica in time.</exception>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public bool Observe(MutationToken token, ReplicateTo replicateTo, PersistTo persistTo)
{
var keyMapper = (VBucketKeyMapper)_configInfo.GetKeyMapper();
var p = new ObserveParams
{
ReplicateTo = replicateTo,
PersistTo = persistTo,
Token = token,
VBucket = keyMapper[token.VBucketId]
};
p.CheckConfiguredReplicas();
var op = new ObserveSeqno(p.Token, _transcoder, _timeout);
do
{
var master = p.VBucket.LocatePrimary();
var result = master.Send(op);
var osr = result.Value;
p.CheckMutationLost(osr);
p.CheckPersisted(osr);
if (p.IsDurabilityMet())
{
return true;
}
if (CheckReplicas(p, op))
{
return true;
}
//prepare for another attempt
op = (ObserveSeqno)op.Clone();
p.Reset();
} while (!op.TimedOut());
return false;
}
public async Task<bool> ObserveAsync(string key, ulong cas, bool deletion, ReplicateTo replicateTo,
PersistTo persistTo, CancellationTokenSource cts)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket)keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
observeParams.CheckConfiguredReplicas();
var persisted = await CheckPersistToAsync(observeParams).ContinueOnAnyContext();
var replicated = await CheckReplicasAsync(observeParams).ContinueOnAnyContext();
if (persisted && replicated)
{
return true;
}
return await ObserveEveryAsync(async p =>
{
Log.DebugFormat("trying again: {0}", key);
persisted = await CheckPersistToAsync(observeParams).ContinueOnAnyContext();
replicated = await CheckReplicasAsync(observeParams).ContinueOnAnyContext();
return persisted & replicated;
}, observeParams, _interval, cts.Token);
}
/// <summary>
/// Checks the replicas to see if the key has met the durability constraints defined by the caller.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="operation">The operation observe operation reference; will be cloned if reused.</param>
/// <param name="replicateTo">The replication durability that must be met.</param>
/// <param name="persistTo">The persistence durbaility that must be met.</param>
/// <returns></returns>
bool CheckReplicas(ObserveParams observeParams, Observe operation, ReplicateTo replicateTo, PersistTo persistTo)
{
//Get the candidate replicas, if none are defined that match the specified durability return false.
var replicas = GetReplicas(observeParams.VBucket, replicateTo, persistTo);
if (replicas.Count < (int)replicateTo)
{
return false;
}
//Check each replica to see if has met the durability constraints specified. A mutation means we failed.
var mutated = replicas.All(index => CheckReplica(observeParams, operation, index));
return observeParams.IsDurabilityMet() && !mutated;
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task<bool> ObserveEvery(Func<ObserveParams, bool> observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
if (observe(observeParams))
{
return true;
}
var task = Task.Delay(interval, cancellationToken);
try
{
await task;
}
catch (TaskCanceledException)
{
return false;
}
}
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task<bool> ObserveEveryAsync(Func<ObserveParams, Task<bool>> observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return true;
}
//delay for the interval - will throw TaskCancellationException if the token times out
await Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
}
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task <bool> ObserveEvery(Func <ObserveParams, Task <bool> > observe, ObserveParams observeParams, int interval, ObserveSeqno op, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return(true);
}
var task = Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
try
{
await task;
}
catch (TaskCanceledException)
{
return(false);
}
//prepare for a second attempt
op = (ObserveSeqno)op.Clone();
observeParams.Reset();
}
}
/// <summary>
/// Performs an observe event on the durability requirements specified on a key asynchronously
/// </summary>
/// <param name="key">The key to observe.</param>
/// <param name="cas">The 'Check and Set' value of the key.</param>
/// <param name="deletion">True if this is a delete operation.</param>
/// <param name="replicateTo">The number of replicas that the key must be replicated to to satisfy the durability constraint.</param>
/// <param name="persistTo">The number of replicas that the key must be persisted to to satisfy the durability constraint.</param>
/// <returns> A <see cref="Task{bool}"/> representing the aynchronous operation.</returns>
/// <exception cref="ReplicaNotConfiguredException">Thrown if the number of replicas requested
/// in the ReplicateTo parameter does not match the # of replicas configured on the server.</exception>
public async Task<bool> ObserveAsync(string key, ulong cas, bool deletion, ReplicateTo replicateTo,
PersistTo persistTo)
{
var criteria = GetDurabilityCriteria(deletion);
var keyMapper = _configInfo.GetKeyMapper();
var vBucket = (IVBucket) keyMapper.MapKey(key);
var observeParams = new ObserveParams
{
Cas = cas,
Criteria = criteria,
Key = key,
PersistTo = persistTo,
ReplicateTo = replicateTo,
VBucket = vBucket
};
observeParams.CheckConfiguredReplicas();
var operation = new Observe(key, vBucket, _transcoder, (uint)_timeout);
//Used to terminate the loop at the specific timeout
using (var cts = new CancellationTokenSource(_timeout))
{
//perform the observe operation at the set interval and terminate if not successful by the timeout
var task = await ObserveEvery(async p =>
{
//check the master for persistence to disk
IServer master;
var attempts = 0;
while ((master = p.VBucket.LocatePrimary()) == null)
{
if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
await Task.Delay((int) Math.Pow(2, attempts));
}
var result = master.Send(operation);
Log.Debug(m => m("Master {0} - {1}", master.EndPoint, result.Value));
var state = result.Value;
if (state.KeyState == p.Criteria.PersistState)
{
Interlocked.Increment(ref p.PersistedToCount);
}
//Key mutation detected so fail
if (p.HasMutated(state.Cas))
{
return false;
}
//Check if durability requirements have been met
if (p.IsDurabilityMet())
{
return true;
}
//Run the durability requirement check on each replica
var tasks = new List<Task<bool>>();
var replicas = p.GetReplicas();
replicas.ForEach(x => tasks.Add(CheckReplicaAsync(p, operation, x)));
//Wait for all tasks to finish
await Task.WhenAll(tasks.ToArray()).ContinueOnAnyContext();
var mutated = tasks.All(subtask => subtask.Result);
return p.IsDurabilityMet() && !mutated;
}, observeParams, operation, _interval, cts.Token).ContinueOnAnyContext();
return task;
}
}
/// <summary>
/// Checks the replicas for durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="op">The op.</param>
/// <returns></returns>
bool CheckReplicas(ObserveParams observeParams, ObserveSeqno op)
{
var replicas = observeParams.GetReplicas();
return(replicas.Any(replicaId => CheckReplica(observeParams, op, replicaId)));
}
/// <summary>
/// Checks the replicas to see if the key has met the durability constraints defined by the caller.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="operation">The operation observe operation reference; will be cloned if reused.</param>
/// <returns></returns>
bool CheckReplicas(ObserveParams observeParams, Observe operation)
{
//Get the candidate replicas, if none are defined that match the specified durability return false.
var replicas = observeParams.GetReplicas();
//Check each replica to see if has met the durability constraints specified. A mutation means we failed.
var mutated = replicas.All(index => CheckReplica(observeParams, operation, index));
return observeParams.IsDurabilityMet() && !mutated;
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
private async Task <bool> ObserveEveryAsync(Func <ObserveParams, Task <bool> > observe, ObserveParams observeParams, int interval, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return(true);
}
//delay for the interval - will throw TaskCancellationException if the token times out
await Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
}
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="observeParams">The <see cref="ObserveParams"/> object.</param>
/// <param name="operation">The Observe operation.</param>
/// <param name="replicaIndex">The replicaIndex of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task<bool> CheckReplicaAsync(ObserveParams observeParams, Observe operation, int replicaIndex)
{
Log.Debug(m=>m("checking replica {0}", replicaIndex));
if (observeParams.IsDurabilityMet()) return true;
//clone the operation since we already checked the primary and we want to maintain internal state (opaque, timer, etc)
operation = (Observe)operation.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaIndex);
var result = await Task.Run(()=>replica.Send(operation)).ContinueOnAnyContext();
Log.Debug(m=>m("Replica {0} - {1} [0]", replica.EndPoint, result.Value, replicaIndex));
var state = result.Value;
if (state.KeyState == observeParams.Criteria.PersistState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
Interlocked.Increment(ref observeParams.PersistedToCount);
}
else if (state.KeyState == observeParams.Criteria.ReplicateState)
{
Interlocked.Increment(ref observeParams.ReplicatedToCount);
}
return !observeParams.HasMutated(state.Cas);
}
private async Task<bool> CheckReplicaAsync(ObserveParams observeParams, int replicaId)
{
var op = new ObserveSeqno(observeParams.Token, _clusterController.Transcoder, _timeout);
observeParams.Operation = op;
var tcs = new TaskCompletionSource<IOperationResult<ObserveSeqnoResponse>>();
op.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(op.Opaque, op);
Log.Debug(m => m("checking replica {0} - opaque: {1}", replicaId, op.Opaque));
var replica = observeParams.VBucket.LocateReplica(replicaId);
await replica.SendAsync(op).ContinueOnAnyContext();
var response = await tcs.Task.ContinueOnAnyContext();
observeParams.CheckMutationLost(response);
observeParams.CheckPersisted(response);
observeParams.CheckReplicated(response);
return observeParams.IsDurabilityMet();
}
/// <summary>
/// Observes a set of keys at a specified interval and timeout.
/// </summary>
/// <param name="observe">The func to call at the specific interval</param>
/// <param name="observeParams">The parameters to pass in.</param>
/// <param name="interval">The interval to check.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use to terminate the observation at the specified timeout.</param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task<bool> ObserveEvery(Func<ObserveParams, Task<bool>> observe, ObserveParams observeParams, int interval, ObserveSeqno op, CancellationToken cancellationToken)
{
while (true)
{
var result = await observe(observeParams).ContinueOnAnyContext();
if (result)
{
return true;
}
var task = Task.Delay(interval, cancellationToken).ContinueOnAnyContext();
try
{
await task;
}
catch (TaskCanceledException)
{
return false;
}
//prepare for a second attempt
op = (ObserveSeqno)op.Clone();
observeParams.Reset();
}
}
/// <summary>
/// Checks a replica for durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="op">The op.</param>
/// <param name="replicaId">The replica identifier.</param>
/// <returns></returns>
private bool CheckReplica(ObserveParams observeParams, ObserveSeqno op, int replicaId)
{
var cloned = (ObserveSeqno)op.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaId);
var result = replica.Send(cloned);
observeParams.CheckMutationLost(result);
observeParams.CheckPersisted(result);
observeParams.CheckReplicated(result);
return observeParams.IsDurabilityMet();
}
/// <summary>
/// Checks the replicas for durability constraints.
/// </summary>
/// <param name="observeParams">The observe parameters.</param>
/// <param name="op">The op.</param>
/// <returns></returns>
bool CheckReplicas(ObserveParams observeParams, ObserveSeqno op)
{
var replicas = observeParams.GetReplicas();
return replicas.Any(replicaId => CheckReplica(observeParams, op, replicaId));
}
/// <summary>
/// Asynchronously checks the replications status of a key.
/// </summary>
/// <param name="op">The <see cref="ObserveParams"/> object.</param>
/// <param name="replicaIndex">The index of the replica within the <see cref="IVBucket"/></param>
/// <returns>True if the durability requirements specified by <see cref="PersistTo"/> and <see cref="ReplicateTo"/> have been satisfied.</returns>
static async Task<bool> CheckReplica(ObserveParams op, int replicaIndex)
{
Log.Debug(m=>m("checking replica {0}", replicaIndex));
if (op.IsDurabilityMet()) return true;
var replica = op.VBucket.LocateReplica(replicaIndex);
var result = await Task.Run(()=>replica.Send(new Observe(op.Key, op.VBucket, new AutoByteConverter())));
Log.Debug(m=>m("Replica {0} - {1} [0]", replica.EndPoint, result.Value, replicaIndex));
var state = result.Value;
if (state.KeyState == op.Criteria.PersistState)
{
Interlocked.Increment(ref op.ReplicatedToCount);
Interlocked.Increment(ref op.PersistedToCount);
}
else if (state.KeyState == op.Criteria.ReplicateState)
{
Interlocked.Increment(ref op.ReplicatedToCount);
}
return op.IsDurabilityMet() && !op.HasMutated(state.Cas);
}
static Task<bool> CheckReplicaAsync(ObserveParams observeParams, ObserveSeqno op, int replicaId)
{
var cloned = (ObserveSeqno)op.Clone();
var replica = observeParams.VBucket.LocateReplica(replicaId);
var result = replica.Send(cloned);
observeParams.CheckMutationLost(result.Value);
observeParams.CheckPersisted(result.Value);
observeParams.CheckReplicated(result.Value);
return Task.FromResult(observeParams.IsDurabilityMet());
}
private async Task<bool> CheckPersistToAsync(ObserveParams observeParams)
{
if (observeParams.PersistTo == PersistTo.Zero) return true;
var op = new ObserveSeqno(observeParams.Token, _clusterController.Transcoder, _timeout);
observeParams.Operation = op;
var tcs = new TaskCompletionSource<IOperationResult<ObserveSeqnoResponse>>();
op.Completed = CallbackFactory.CompletedFuncForRetry(_pending, _clusterController, tcs);
_pending.TryAdd(op.Opaque, op);
var server = await GetServerAsync(observeParams);
await server.SendAsync(op).ContinueOnAnyContext();
var response = await tcs.Task.ContinueOnAnyContext();
observeParams.CheckMutationLost(response);
observeParams.CheckPersisted(response);
return observeParams.IsDurabilityMet();
}
