/// <summary>
/// Sends a <see cref="IOperation{T}"/> to the Couchbase Server using the Memcached protocol.
/// </summary>
/// <typeparam name="T">The Type of the body of the request.</typeparam>
/// <param name="operation">The <see cref="IOperation{T}"/> to send.</param>
/// <returns>An <see cref="IOperationResult"/> with the status of the request.</returns>
public override IOperationResult <T> SendWithRetry <T>(IOperation <T> operation)
{
IOperationResult <T> operationResult = new OperationResult <T>
{
Success = false, OpCode = operation.OperationCode
};
var parentSpan = Tracer.StartParentSpan(operation, ConfigInfo.BucketName);
try
{
do
{
var server = GetServer(operation.Key);
if (server == null)
{
continue;
}
operationResult = server.Send(operation);
if (operationResult.Success)
{
Log.Debug("Operation succeeded {0} for key {1}", operation.Attempts, operation.Key);
break;
}
if (operation.CanRetry() && operationResult.ShouldRetry())
{
var result = operationResult;
Log.Debug("Operation retry {0} for key {1}. Reason: {2}", operation.Attempts,
operation.Key, result.Message);
// Get retry timeout, uses default timeout if no retry stratergy available
Thread.Sleep(operation.GetRetryTimeout(VBucketRetrySleepTime));
}
else
{
((OperationResult)operationResult).SetException();
Log.Debug("Operation doesn't support retries for key {0}", operation.Key);
break;
}
} while (operation.Attempts++ < operation.MaxRetries && !operationResult.Success);
if (!operationResult.Success)
{
if (operation.TimedOut())
{
const string msg = "The operation has timed out.";
((OperationResult)operationResult).Message = msg;
((OperationResult)operationResult).Status = ResponseStatus.OperationTimeout;
}
LogFailure(operation, operationResult);
}
}
finally
{
parentSpan.Finish();
}
return(operationResult);
}
/// <summary>
/// Checks the <see cref="IOperation"/> to see if it supports retries and then checks the <see cref="IOperationResult"/>
/// to see if the error or server response supports retries.
/// </summary>
/// <typeparam name="T">The Type of the body of the request.</typeparam>
/// <param name="operationResult">The <see cref="IOperationResult"/> to check from the server.</param>
/// <param name="operation">The <see cref="IOperation"/> to check to see if it supports retries. Not all operations support retries.</param>
/// <returns></returns>
public bool CanRetryOperation(IOperationResult operationResult, IOperation operation)
{
var responseStatus = operationResult.Status;
if (responseStatus == ResponseStatus.VBucketBelongsToAnotherServer)
{
return(CheckForConfigUpdates(operation));
}
return(operation.CanRetry() && operationResult.ShouldRetry());
}
/// <summary>
/// Sends a <see cref="IOperation{T}"/> to the Couchbase Server using the Memcached protocol.
/// </summary>
/// <typeparam name="T">The Type of the body of the request.</typeparam>
/// <param name="operation">The <see cref="IOperation{T}"/> to send.</param>
/// <returns>An <see cref="IOperationResult"/> with the status of the request.</returns>
public override IOperationResult <T> SendWithRetry <T>(IOperation <T> operation)
{
IOperationResult <T> operationResult = new OperationResult <T>
{
Success = false, OpCode = operation.OperationCode
};
do
{
var server = GetServer(operation.Key);
if (server == null)
{
continue;
}
operationResult = server.Send(operation);
if (operationResult.Success)
{
Log.Debug("Operation succeeded {0} for key {1}", operation.Attempts, operation.Key);
break;
}
if (operation.CanRetry() && operationResult.ShouldRetry())
{
var result = operationResult;
Log.Debug("Operation retry {0} for key {1}. Reason: {2}", operation.Attempts,
operation.Key, result.Message);
Thread.Sleep(VBucketRetrySleepTime);
}
else
{
((OperationResult)operationResult).SetException();
Log.Debug("Operation doesn't support retries for key {0}", operation.Key);
break;
}
} while (operation.Attempts++ < operation.MaxRetries && !operationResult.Success);
if (!operationResult.Success)
{
if (operation.TimedOut())
{
const string msg = "The operation has timed out.";
((OperationResult)operationResult).Message = msg;
((OperationResult)operationResult).Status = ResponseStatus.OperationTimeout;
}
LogFailure(operation, operationResult);
}
return(operationResult);
}
/// <summary>
/// Sends a <see cref="IOperation{T}"/> to the Couchbase Server using the Memcached protocol.
/// </summary>
/// <typeparam name="T">The Type of the body of the request.</typeparam>
/// <param name="operation">The <see cref="IOperation{T}"/> to send.</param>
/// <returns>An <see cref="IOperationResult"/> with the status of the request.</returns>
public override IOperationResult SendWithRetry(IOperation operation)
{
IOperationResult operationResult = new OperationResult {
Success = false
};
do
{
var server = GetServer(operation.Key);
if (server == null)
{
continue;
}
operationResult = server.Send(operation);
if (operationResult.Success)
{
Log.Debug(m => m("Operation succeeded {0} for key {1}", operation.Attempts, operation.Key));
break;
}
if (operation.CanRetry() && operationResult.ShouldRetry())
{
Log.Debug(m => m("Operation retry {0} for key {1}. Reason: {2}", operation.Attempts,
operation.Key, operationResult.Message));
}
else
{
Log.Debug(m => m("Operation doesn't support retries for key {0}", operation.Key));
break;
}
} while (operation.Attempts++ < operation.MaxRetries && !operationResult.Success);
if (!operationResult.Success)
{
if (operation.TimedOut())
{
const string msg = "The operation has timed out.";
((OperationResult)operationResult).Message = msg;
((OperationResult)operationResult).Status = ResponseStatus.OperationTimeout;
}
const string msg1 = "Operation for key {0} failed after {1} retries for opaque{4}. Reason: {5}";
Log.Debug(m => m(msg1, operation.Key, operation.Attempts, operation.Opaque, operationResult.Message));
}
return(operationResult);
}
/// <summary>
/// Executes an operation until it either succeeds, reaches a non-retriable state, or times out.
/// </summary>
/// <typeparam name="T">The Type of the <see cref="IOperation"/>'s value.</typeparam>
/// <param name="execute">A delegate that contains the send logic.</param>
/// <param name="operation">The <see cref="IOperation"/> to execiute.</param>
/// <param name="configInfo">The <see cref="IConfigInfo"/> that represents the logical topology of the cluster.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for timing out the request.</param>
/// An <see cref="Task{IOperationResult}"/> object representing the asynchrobous operation.
public virtual async Task <IOperationResult <T> > RetryOperationEveryAsync <T>(
Func <IOperation <T>, IConfigInfo, Task <IOperationResult <T> > > execute,
IOperation <T> operation,
IConfigInfo configInfo,
CancellationToken cancellationToken)
{
while (true)
{
var result = await execute(operation, configInfo).ConfigureAwait(false);
if (result.Success || operation.TimedOut())
{
if (operation.TimedOut())
{
const string msg = "The operation has timed out. Retried [{0}] times.";
((OperationResult)result).Message = string.Format(msg, operation.Attempts);
((OperationResult)result).Status = ResponseStatus.OperationTimeout;
}
return(result);
}
if (!result.IsNmv() && !operation.CanRetry())
{
return(result);
}
operation.Attempts++;
var sleepTime = (int)Math.Pow(2, operation.Attempts * 2);
var task = Task.Delay(sleepTime, cancellationToken).ConfigureAwait(false);
try
{
await task;
}
catch (TaskCanceledException)
{
const string msg = "The operation has timed out. Retried [{0}] times.";
((OperationResult)result).Message = string.Format(msg, operation.Attempts);
((OperationResult)result).Status = ResponseStatus.OperationTimeout;
return(result);
}
}
}
/// <summary>
/// Executes an operation until it either succeeds, reaches a non-retriable state, or times out.
/// </summary>
/// <param name="execute">A delegate that contains the send logic.</param>
/// <param name="operation">The <see cref="IOperation"/> to execiute.</param>
/// <param name="configInfo">The <see cref="IConfigInfo"/> that represents the logical topology of the cluster.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> for timing out the request.</param>
/// An <see cref="Task{IOperationResult}" /> object representing the asynchrobous operation.
public virtual async Task <IOperationResult> RetryOperationEveryAsync(
Func <IOperation, IConfigInfo, Task <IOperationResult> > execute,
IOperation operation,
IConfigInfo configInfo,
CancellationToken cancellationToken)
{
while (true)
{
var result = await execute(operation, configInfo).ContinueOnAnyContext();
if (result.Success || operation.TimedOut())
{
if (operation.TimedOut())
{
const string msg = "The operation has timed out. Retried [{0}] times.";
((OperationResult)result).Message = string.Format(msg, operation.Attempts);
((OperationResult)result).Status = ResponseStatus.OperationTimeout;
}
return(result);
}
if (!result.IsNmv() || !operation.CanRetry())
{
return(result);
}
operation.Attempts++;
var task = Task.Delay(VBucketRetrySleepTime, cancellationToken).ContinueOnAnyContext();
try
{
await task;
}
catch (TaskCanceledException)
{
const string msg = "The operation has timed out. Retried [{0}] times.";
((OperationResult)result).Message = string.Format(msg, operation.Attempts);
((OperationResult)result).Status = ResponseStatus.OperationTimeout;
return(result);
}
}
}
