/// <summary>
/// Gets the retry decision based on the request error
/// </summary>
internal static RetryDecisionWithReason GetRetryDecisionWithReason(
IRequestError error, IExtendedRetryPolicy policy, IStatement statement, Configuration config, int retryCount)
{
var ex = error.Exception;
if (ex is SocketException || ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException ||
ex is OperationTimedOutException)
{
if (ex is SocketException exception)
{
RequestExecution.Logger.Verbose("Socket error " + exception.SocketErrorCode);
}
// For PREPARE requests, retry on next host
var decision = statement == null && ex is OperationTimedOutException
? RetryDecision.Retry(null, false)
: policy.OnRequestError(statement, config, ex, retryCount);
return(new RetryDecisionWithReason(decision, RequestExecution.GetErrorType(error)));
}
if (ex is ReadTimeoutException e)
{
return(new RetryDecisionWithReason(
policy.OnReadTimeout(
statement,
e.ConsistencyLevel,
e.RequiredAcknowledgements,
e.ReceivedAcknowledgements,
e.WasDataRetrieved,
retryCount),
RequestErrorType.ReadTimeOut
));
}
if (ex is WriteTimeoutException e1)
{
return(new RetryDecisionWithReason(
policy.OnWriteTimeout(
statement,
e1.ConsistencyLevel,
e1.WriteType,
e1.RequiredAcknowledgements,
e1.ReceivedAcknowledgements,
retryCount),
RequestErrorType.WriteTimeOut
));
}
if (ex is UnavailableException e2)
{
return(new RetryDecisionWithReason(
policy.OnUnavailable(statement, e2.Consistency, e2.RequiredReplicas, e2.AliveReplicas, retryCount),
RequestErrorType.Unavailable
));
}
// Any other Exception just throw it
return(new RetryDecisionWithReason(RetryDecision.Rethrow(), RequestExecution.GetErrorType(error)));
}
/// <summary>
/// Gets the retry decision based on the exception from Cassandra
/// </summary>
public static RetryDecision GetRetryDecision(Exception ex, IRetryPolicy policy, IStatement statement, int retryCount)
{
var decision = RetryDecision.Rethrow();
if (ex is SocketException)
{
decision = RetryDecision.Retry(null, false);
}
else if (ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException)
{
decision = RetryDecision.Retry(null, false);
}
else if (ex is ReadTimeoutException)
{
var e = (ReadTimeoutException)ex;
decision = policy.OnReadTimeout(statement, e.ConsistencyLevel, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, e.WasDataRetrieved, retryCount);
}
else if (ex is WriteTimeoutException)
{
var e = (WriteTimeoutException)ex;
decision = policy.OnWriteTimeout(statement, e.ConsistencyLevel, e.WriteType, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, retryCount);
}
else if (ex is UnavailableException)
{
var e = (UnavailableException)ex;
decision = policy.OnUnavailable(statement, e.Consistency, e.RequiredReplicas, e.AliveReplicas, retryCount);
}
return(decision);
}
/// <summary>
/// Defines whether to retry and at which consistency level on a write timeout.
/// <p> This method triggers a maximum of one retry. If <code>writeType ==
/// WriteType.BATCH_LOG</code>, the write is retried with the initial consistency
/// level. If <code>writeType == WriteType.UNLOGGED_BATCH</code> and at least one
/// replica acknowleged, the write is retried with a lower consistency level
/// (with unlogged batch, a write timeout can <b>always</b> mean that part of the
/// batch haven't been persisted at' all, even if <code>receivedAcks > 0</code>).
/// For other <code>writeType</code>, if we know the write has been persisted on
/// at least one replica, we ignore the exception. Otherwise, an exception is
/// thrown.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the write that timeouted.
/// </param>
/// <param name="writeType"> the type of the write that timeouted. </param>
/// <param name="requiredAcks"> the number of acknowledgments that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedAcks"> the number of acknowledgments that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnWriteTimeout(Query query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
switch (writeType)
{
case "SIMPLE":
case "BATCH":
// Since we provide atomicity there is no point in retrying
return(RetryDecision.Ignore());
case "COUNTER":
// We should not retry counters, period!
return(RetryDecision.Ignore());
case "UNLOGGED_BATCH":
// Since only part of the batch could have been persisted,
// retry with whatever consistency should allow to persist all
return(MaxLikelyToWorkCl(receivedAcks));
case "BATCH_LOG":
return(RetryDecision.Retry(cl));
}
return(RetryDecision.Rethrow());
}
/// <inheritdoc />
public RetryDecision OnUnavailable(
IStatement query,
ConsistencyLevel cl,
int requiredReplica,
int aliveReplica,
int nbRetry)
{
return(RetryDecision.Rethrow());
}
/// <summary>
/// Defines whether to retry and at which consistency level on a write timeout.
/// <p> This method triggers a maximum of one retry, and only in the case of a
/// <c>WriteType.BATCH_LOG</c> write. The reasoning for the retry in that
/// case is that write to the distributed batch log is tried by the coordinator
/// of the write against a small subset of all the node alive in the local
/// datacenter. Hence, a timeout usually means that none of the nodes in that
/// subset were alive but the coordinator hasn't' detected them as dead. By the
/// time we get the timeout the dead nodes will likely have been detected as dead
/// and the retry has thus a high change of success.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the write that timeouted.
/// </param>
/// <param name="writeType"> the type of the write that timeouted. </param>
/// <param name="requiredAcks"> the number of acknowledgments that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedAcks"> the number of acknowledgments that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns><c>RetryDecision.retry(cl)</c> if no retry attempt has yet
/// been tried and <c>writeType == WriteType.BATCH_LOG</c>,
/// <c>RetryDecision.rethrow()</c> otherwise.</returns>
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks,
int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
// If the batch log write failed, retry the operation as this might just be we were unlucky at picking candidtes
return(writeType == "BATCH_LOG" ? RetryDecision.Retry(cl) : RetryDecision.Rethrow());
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
return(receivedResponses >= requiredResponses && !dataRetrieved
? RetryDecision.Retry(cl)
: RetryDecision.Rethrow());
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(null));
}
}
public RetryDecision onWriteTimeout(Query query, ConsistencyLevel cl,
String writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
int hour = DateTime.Now.Hour;
if (hour >= nonRetryStartHour && hour <= nonRetryEndHour)
{
return(RetryDecision.Rethrow());
}
return(policy.OnWriteTimeout(query, cl, writeType, requiredAcks,
receivedAcks, nbRetry));
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
_action?.Invoke(nbRetry).GetAwaiter().GetResult();
if (nbRetry > _maxRetries)
{
return(RetryDecision.Rethrow());
}
Interlocked.Increment(ref RequestErrorCounter);
return(RetryDecision.Retry(null, true));
}
public RetryDecision onUnavailable(Query query, ConsistencyLevel cl,
int requiredReplica, int aliveReplica, int nbRetry)
{
int hour = DateTime.Now.Hour;
if (hour >= nonRetryStartHour && hour <= nonRetryEndHour)
{
return(RetryDecision.Rethrow());
}
return(policy.OnUnavailable(query, cl, requiredReplica, aliveReplica,
nbRetry));
}
public RetryDecision onReadTimeout(Query query, ConsistencyLevel cl,
int requiredResponses, int receivedResponses,
Boolean dataRetrieved, int nbRetry)
{
int hour = DateTime.Now.Hour;
if (hour >= nonRetryStartHour && hour <= nonRetryEndHour)
{
return(RetryDecision.Rethrow());
}
return(policy.OnReadTimeout(query, cl, requiredResponses,
receivedResponses, dataRetrieved, nbRetry));
}
/// <inheritdoc />
public RetryDecision OnRequestError(
IStatement statement,
Configuration config,
Exception ex,
int nbRetry)
{
if (ex is OverloadedException ||
(ex is OperationTimedOutException && statement.IsIdempotent.GetValueOrDefault(false)))
{
return(this.GenerateRetryDecision(nbRetry, statement.ConsistencyLevel));
}
return(RetryDecision.Rethrow());
}
/// <inheritdoc />
public RetryDecision OnWriteTimeout(
IStatement query,
ConsistencyLevel cl,
string writeType,
int requiredAcks,
int receivedAcks,
int nbRetry)
{
if (query != null && query.IsIdempotent == true)
{
return(this.GenerateRetryDecision(nbRetry, cl));
}
return(RetryDecision.Rethrow());
}
private RetryDecision GenerateRetryDecision(int nbRetry, ConsistencyLevel?cl)
{
if (this.maxRetryCount == -1)
{
Thread.Sleep(CosmosDBMultipleRetryPolicy.FixedBackOffTimeMs);
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
else if (nbRetry < this.maxRetryCount)
{
Thread.Sleep(CosmosDBMultipleRetryPolicy.GrowingBackOffTimeMs * nbRetry);
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
return(RetryDecision.Rethrow());
}
/// <inheritdoc />
public RetryDecision OnReadTimeout(
IStatement query,
ConsistencyLevel cl,
int requiredResponses,
int receivedResponses,
bool dataRetrieved,
int nbRetry)
{
if (nbRetry < this.maxRetryCount)
{
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
return(RetryDecision.Rethrow());
}
/// <summary>
/// Defines whether to retry and at which consistency level on a read timeout.
/// <p> This method triggers a maximum of one retry. If less replica responsed
/// than required by the consistency level (but at least one replica did
/// respond), the operation is retried at a lower consistency level. If enough
/// replica responded but data was not retrieve, the operation is retried with
/// the initial consistency level. Otherwise, an exception is thrown.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the read that timeouted.
/// </param>
/// <param name="requiredResponses"> the number of responses that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedResponses"> the number of responses that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="dataRetrieved"> whether actual data (by opposition to data
/// checksum) was present in the received responses. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnReadTimeout(Query query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
if (receivedResponses < requiredResponses)
{
// Tries the biggest CL that is expected to work
return(MaxLikelyToWorkCl(receivedResponses));
}
return(!dataRetrieved?RetryDecision.Retry(cl) : RetryDecision.Rethrow());
}
private static RetryDecision MaxLikelyToWorkCl(int knownOk)
{
if (knownOk >= 3)
{
return(RetryDecision.Retry(ConsistencyLevel.Three));
}
if (knownOk >= 2)
{
return(RetryDecision.Retry(ConsistencyLevel.Two));
}
if (knownOk >= 1)
{
return(RetryDecision.Retry(ConsistencyLevel.One));
}
return(RetryDecision.Rethrow());
}
/// <summary>
/// Gets the retry decision based on the exception from Cassandra
/// </summary>
public static RetryDecision GetRetryDecision(Exception ex, IExtendedRetryPolicy policy, IStatement statement,
Configuration config, int retryCount)
{
if (ex is SocketException)
{
Logger.Verbose("Socket error " + ((SocketException)ex).SocketErrorCode);
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException)
{
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is ReadTimeoutException)
{
var e = (ReadTimeoutException)ex;
return(policy.OnReadTimeout(statement, e.ConsistencyLevel, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, e.WasDataRetrieved, retryCount));
}
if (ex is WriteTimeoutException)
{
var e = (WriteTimeoutException)ex;
return(policy.OnWriteTimeout(statement, e.ConsistencyLevel, e.WriteType, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, retryCount));
}
if (ex is UnavailableException)
{
var e = (UnavailableException)ex;
return(policy.OnUnavailable(statement, e.Consistency, e.RequiredReplicas, e.AliveReplicas, retryCount));
}
if (ex is OperationTimedOutException)
{
if (statement == null)
{
// For PREPARE requests, retry on next host
return(RetryDecision.Retry(null, false));
}
// Delegate on retry policy
return(policy.OnRequestError(statement, config, ex, retryCount));
}
// Any other Exception just throw it
return(RetryDecision.Rethrow());
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
Interlocked.Increment(ref RequestErrorConter);
return(RetryDecision.Rethrow());
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
return(RetryDecision.Rethrow());
}
/// <summary>
/// Defines whether to retry and at which consistency level on an unavailable
/// exception. <p> This method triggers a maximum of one retry. If at least one
/// replica is know to be alive, the operation is retried at a lower consistency
/// level.</p>
/// </summary>
/// <param name="query"> the original query for which the consistency level
/// cannot be achieved. </param>
/// <param name="cl"> the original consistency level for the operation. </param>
/// <param name="requiredReplica"> the number of replica that should have been
/// (known) alive for the operation to be attempted. </param>
/// <param name="aliveReplica"> the number of replica that were know to be alive
/// by the coordinator of the operation. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnUnavailable(Query query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
return(nbRetry != 0 ? RetryDecision.Rethrow() : MaxLikelyToWorkCl(aliveReplica));
// Tries the biggest CL that is expected to work
}
public RetryDecision OnReadTimeout(Query query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
return(RetryDecision.Rethrow());
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
Interlocked.Increment(ref UnavailableCounter);
return(RetryDecision.Rethrow());
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Interlocked.Increment(ref WriteTimeoutCounter);
return(RetryDecision.Rethrow());
}