/// <summary>
/// Gets the retry decision based on the request error
/// </summary>
internal static RetryDecisionWithReason GetRetryDecisionWithReason(
IRequestError error, IExtendedRetryPolicy policy, IStatement statement, Configuration config, int retryCount)
{
var ex = error.Exception;
if (ex is SocketException || ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException ||
ex is OperationTimedOutException)
{
if (ex is SocketException exception)
{
RequestExecution.Logger.Verbose("Socket error " + exception.SocketErrorCode);
}
// For PREPARE requests, retry on next host
var decision = statement == null && ex is OperationTimedOutException
? RetryDecision.Retry(null, false)
: policy.OnRequestError(statement, config, ex, retryCount);
return(new RetryDecisionWithReason(decision, RequestExecution.GetErrorType(error)));
}
if (ex is ReadTimeoutException e)
{
return(new RetryDecisionWithReason(
policy.OnReadTimeout(
statement,
e.ConsistencyLevel,
e.RequiredAcknowledgements,
e.ReceivedAcknowledgements,
e.WasDataRetrieved,
retryCount),
RequestErrorType.ReadTimeOut
));
}
if (ex is WriteTimeoutException e1)
{
return(new RetryDecisionWithReason(
policy.OnWriteTimeout(
statement,
e1.ConsistencyLevel,
e1.WriteType,
e1.RequiredAcknowledgements,
e1.ReceivedAcknowledgements,
retryCount),
RequestErrorType.WriteTimeOut
));
}
if (ex is UnavailableException e2)
{
return(new RetryDecisionWithReason(
policy.OnUnavailable(statement, e2.Consistency, e2.RequiredReplicas, e2.AliveReplicas, retryCount),
RequestErrorType.Unavailable
));
}
// Any other Exception just throw it
return(new RetryDecisionWithReason(RetryDecision.Rethrow(), RequestExecution.GetErrorType(error)));
}
/// <summary>
/// Defines whether to retry and at which consistency level on a write timeout.
/// <p> This method triggers a maximum of one retry. If <code>writeType ==
/// WriteType.BATCH_LOG</code>, the write is retried with the initial consistency
/// level. If <code>writeType == WriteType.UNLOGGED_BATCH</code> and at least one
/// replica acknowleged, the write is retried with a lower consistency level
/// (with unlogged batch, a write timeout can <b>always</b> mean that part of the
/// batch haven't been persisted at' all, even if <code>receivedAcks > 0</code>).
/// For other <code>writeType</code>, if we know the write has been persisted on
/// at least one replica, we ignore the exception. Otherwise, an exception is
/// thrown.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the write that timeouted.
/// </param>
/// <param name="writeType"> the type of the write that timeouted. </param>
/// <param name="requiredAcks"> the number of acknowledgments that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedAcks"> the number of acknowledgments that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnWriteTimeout(Query query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
switch (writeType)
{
case "SIMPLE":
case "BATCH":
// Since we provide atomicity there is no point in retrying
return(RetryDecision.Ignore());
case "COUNTER":
// We should not retry counters, period!
return(RetryDecision.Ignore());
case "UNLOGGED_BATCH":
// Since only part of the batch could have been persisted,
// retry with whatever consistency should allow to persist all
return(MaxLikelyToWorkCl(receivedAcks));
case "BATCH_LOG":
return(RetryDecision.Retry(cl));
}
return(RetryDecision.Rethrow());
}
/// <summary>
/// Gets the retry decision based on the exception from Cassandra
/// </summary>
public static RetryDecision GetRetryDecision(Exception ex, IRetryPolicy policy, IStatement statement, int retryCount)
{
var decision = RetryDecision.Rethrow();
if (ex is SocketException)
{
decision = RetryDecision.Retry(null, false);
}
else if (ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException)
{
decision = RetryDecision.Retry(null, false);
}
else if (ex is ReadTimeoutException)
{
var e = (ReadTimeoutException)ex;
decision = policy.OnReadTimeout(statement, e.ConsistencyLevel, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, e.WasDataRetrieved, retryCount);
}
else if (ex is WriteTimeoutException)
{
var e = (WriteTimeoutException)ex;
decision = policy.OnWriteTimeout(statement, e.ConsistencyLevel, e.WriteType, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, retryCount);
}
else if (ex is UnavailableException)
{
var e = (UnavailableException)ex;
decision = policy.OnUnavailable(statement, e.Consistency, e.RequiredReplicas, e.AliveReplicas, retryCount);
}
return(decision);
}
/// <summary>
/// Defines whether to retry and at which consistency level on a write timeout.
/// <p> This method triggers a maximum of one retry, and only in the case of a
/// <c>WriteType.BATCH_LOG</c> write. The reasoning for the retry in that
/// case is that write to the distributed batch log is tried by the coordinator
/// of the write against a small subset of all the node alive in the local
/// datacenter. Hence, a timeout usually means that none of the nodes in that
/// subset were alive but the coordinator hasn't' detected them as dead. By the
/// time we get the timeout the dead nodes will likely have been detected as dead
/// and the retry has thus a high change of success.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the write that timeouted.
/// </param>
/// <param name="writeType"> the type of the write that timeouted. </param>
/// <param name="requiredAcks"> the number of acknowledgments that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedAcks"> the number of acknowledgments that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns><c>RetryDecision.retry(cl)</c> if no retry attempt has yet
/// been tried and <c>writeType == WriteType.BATCH_LOG</c>,
/// <c>RetryDecision.rethrow()</c> otherwise.</returns>
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks,
int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
// If the batch log write failed, retry the operation as this might just be we were unlucky at picking candidtes
return(writeType == "BATCH_LOG" ? RetryDecision.Retry(cl) : RetryDecision.Rethrow());
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
return(receivedResponses >= requiredResponses && !dataRetrieved
? RetryDecision.Retry(cl)
: RetryDecision.Rethrow());
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(null));
}
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
Interlocked.Increment(ref Count);
if (Interlocked.Read(ref Count) > 1000)
{
return(RetryDecision.Rethrow());
}
else
{
return(RetryDecision.Retry(cl));
}
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
_action?.Invoke(nbRetry).GetAwaiter().GetResult();
if (nbRetry > _maxRetries)
{
return(RetryDecision.Rethrow());
}
Interlocked.Increment(ref RequestErrorCounter);
return(RetryDecision.Retry(null, true));
}
/// <inheritdoc />
public RetryDecision OnReadTimeout(
IStatement query,
ConsistencyLevel cl,
int requiredResponses,
int receivedResponses,
bool dataRetrieved,
int nbRetry)
{
if (nbRetry < this.maxRetryCount)
{
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
return(RetryDecision.Rethrow());
}
private RetryDecision GenerateRetryDecision(int nbRetry, ConsistencyLevel?cl)
{
if (this.maxRetryCount == -1)
{
Thread.Sleep(CosmosDBMultipleRetryPolicy.FixedBackOffTimeMs);
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
else if (nbRetry < this.maxRetryCount)
{
Thread.Sleep(CosmosDBMultipleRetryPolicy.GrowingBackOffTimeMs * nbRetry);
return(RetryDecision.Retry(cl, useCurrentHost: true));
}
return(RetryDecision.Rethrow());
}
/// <summary>
/// Defines whether to retry and at which consistency level on a read timeout.
/// <p> This method triggers a maximum of one retry. If less replica responsed
/// than required by the consistency level (but at least one replica did
/// respond), the operation is retried at a lower consistency level. If enough
/// replica responded but data was not retrieve, the operation is retried with
/// the initial consistency level. Otherwise, an exception is thrown.</p>
/// </summary>
/// <param name="query"> the original query that timeouted. </param>
/// <param name="cl"> the original consistency level of the read that timeouted.
/// </param>
/// <param name="requiredResponses"> the number of responses that were required
/// to achieve the requested consistency level. </param>
/// <param name="receivedResponses"> the number of responses that had been
/// received by the time the timeout exception was raised. </param>
/// <param name="dataRetrieved"> whether actual data (by opposition to data
/// checksum) was present in the received responses. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnReadTimeout(Query query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
if (nbRetry != 0)
{
return(RetryDecision.Rethrow());
}
if (receivedResponses < requiredResponses)
{
// Tries the biggest CL that is expected to work
return(MaxLikelyToWorkCl(receivedResponses));
}
return(!dataRetrieved?RetryDecision.Retry(cl) : RetryDecision.Rethrow());
}
private static RetryDecision MaxLikelyToWorkCl(int knownOk)
{
if (knownOk >= 3)
{
return(RetryDecision.Retry(ConsistencyLevel.Three));
}
if (knownOk >= 2)
{
return(RetryDecision.Retry(ConsistencyLevel.Two));
}
if (knownOk >= 1)
{
return(RetryDecision.Retry(ConsistencyLevel.One));
}
return(RetryDecision.Rethrow());
}
/// <summary>
/// Gets the retry decision based on the exception from Cassandra
/// </summary>
public static RetryDecision GetRetryDecision(Exception ex, IExtendedRetryPolicy policy, IStatement statement,
Configuration config, int retryCount)
{
if (ex is SocketException)
{
Logger.Verbose("Socket error " + ((SocketException)ex).SocketErrorCode);
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is OverloadedException || ex is IsBootstrappingException || ex is TruncateException)
{
return(policy.OnRequestError(statement, config, ex, retryCount));
}
if (ex is ReadTimeoutException)
{
var e = (ReadTimeoutException)ex;
return(policy.OnReadTimeout(statement, e.ConsistencyLevel, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, e.WasDataRetrieved, retryCount));
}
if (ex is WriteTimeoutException)
{
var e = (WriteTimeoutException)ex;
return(policy.OnWriteTimeout(statement, e.ConsistencyLevel, e.WriteType, e.RequiredAcknowledgements, e.ReceivedAcknowledgements, retryCount));
}
if (ex is UnavailableException)
{
var e = (UnavailableException)ex;
return(policy.OnUnavailable(statement, e.Consistency, e.RequiredReplicas, e.AliveReplicas, retryCount));
}
if (ex is OperationTimedOutException)
{
if (statement == null)
{
// For PREPARE requests, retry on next host
return(RetryDecision.Retry(null, false));
}
// Delegate on retry policy
return(policy.OnRequestError(statement, config, ex, retryCount));
}
// Any other Exception just throw it
return(RetryDecision.Rethrow());
}
public void Should_RetryRequestToSameHost_When_ConnectionFailsAndRetryDecisionIsRetrySameHost()
{
var mockSession = Mock.Of <IInternalSession>();
var config = new Configuration();
Mock.Get(mockSession).SetupGet(m => m.Cluster.Configuration).Returns(config);
var mockRequest = Mock.Of <IRequest>();
var mockParent = Mock.Of <IRequestHandler>();
var connection = Mock.Of <IConnection>();
// Setup hosts
var host = new Host(
new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9047),
new ConstantReconnectionPolicy(1));
var validHost = ValidHost.New(
host,
HostDistance.Local);
var secondHost = new Host(
new IPEndPoint(IPAddress.Parse("127.0.0.2"), 9047),
new ConstantReconnectionPolicy(1)); // second host should never be used if test passes
var secondValidHost = ValidHost.New(
secondHost,
HostDistance.Local);
// Setup query plan
Mock.Get(mockParent)
.SetupSequence(m => m.GetNextValidHost(It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.Returns(validHost)
.Returns(secondValidHost);
// Setup retry policy
var exception = new OverloadedException(string.Empty);
Mock.Get(mockParent)
.SetupGet(m => m.RetryPolicy)
.Returns(() =>
Mock.Of <IExtendedRetryPolicy>(a =>
a.OnRequestError(
It.IsAny <IStatement>(), config, exception, 0)
== RetryDecision.Retry(null, true)));
// Setup connection failure
Mock.Get(mockParent)
.Setup(m => m.GetConnectionToValidHostAsync(validHost, It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.ThrowsAsync(exception);
// Setup successful second connection on the same host retry (different method call - ValidateHostAndGetConnectionAsync)
Mock.Get(mockParent)
.Setup(m => m.ValidateHostAndGetConnectionAsync(validHost.Host, It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.ReturnsAsync(connection);
var sut = new ProxyRequestExecution(mockParent, mockSession, mockRequest);
sut.Start(false);
// Validate request is sent
TestHelper.RetryAssert(
() =>
{
Mock.Get(connection).Verify(
c => c.Send(mockRequest, It.IsAny <Action <Exception, Response> >(), It.IsAny <int>()),
Times.Once);
});
// Validate that there were 2 connection attempts (1 with each method)
Mock.Get(mockParent).Verify(
m => m.GetConnectionToValidHostAsync(validHost, It.IsAny <Dictionary <IPEndPoint, Exception> >()),
Times.Once);
Mock.Get(mockParent).Verify(
m => m.ValidateHostAndGetConnectionAsync(validHost.Host, It.IsAny <Dictionary <IPEndPoint, Exception> >()),
Times.Once);
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
return(RetryDecision.Retry(ConsistencyLevel.One, false));
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
return(RetryDecision.Retry(ConsistencyLevel.One, false));
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved,
int nbRetry)
{
return(RetryDecision.Retry(ConsistencyLevel.One, false));
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Task.Delay(50).Wait();
return(RetryDecision.Retry(cl));
}
