public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
return(RetryDecision.Retry(ConsistencyLevel.One, false));
}
private static ConsistencyLevel CL(ConsistencyLevel cl, RetryDecision decision)
{
return decision.RetryConsistencyLevel ?? cl;
}
public Task <RetryDecision> ShouldRetryAsync(Exception exception, OperationContext operationContext)
{
this.RetryCallCount++;
return(Task.FromResult(RetryDecision.RetryWithDelay(TimeSpan.FromSeconds(0))));
}
public RetryDecision OnUnavailable(Query query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
return(RetryDecision.Rethrow());
}
public RetryDecisionWithReason(RetryDecision decision, RequestErrorType reason)
{
Decision = decision;
Reason = reason;
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
WriteTimeoutCounter++;
return(RetryDecision.Ignore());
}
public RetryDecision OnReadTimeout(Query query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
return(RetryDecision.Rethrow());
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
Interlocked.Increment(ref RequestErrorConter);
return(RetryDecision.Rethrow());
}
public void Default(RetryDecision action)
{
_default = action;
}
/// <summary>
/// Executes the request.
/// </summary>
/// <returns>An asynchronous operation of return type <typeparamref name="TResponse"/>.</returns>
public async Task <TResponse> ExecuteRequestAsync()
{
this.hasRequestExecutionStarted = true;
bool shouldRetry;
do
{
shouldRetry = false; //Start every request execution assuming there will be no retry
Task <TResponse> serviceRequestTask = null;
Exception capturedException;
//Participate in cooperative cancellation
this.CancellationToken.ThrowIfCancellationRequested();
try
{
this.ApplyClientRequestIdBehaviorToParams();
serviceRequestTask = this.ExecuteRequestWithCancellationAsync(this.ServiceRequestFunc);
TResponse response = await serviceRequestTask.ConfigureAwait(continueOnCapturedContext : false);
//TODO: It would be nice if we could add to OperationContext.RequestResults here
return(response);
}
catch (Exception e)
{
//If the caught exception was a BatchErrorException, we wrap it in the object model exception type
BatchException batchException = null;
if (e is BatchErrorException)
{
batchException = new BatchException(e as BatchErrorException);
}
Exception wrappedException = batchException ?? e;
if (this.RetryPolicy != null &&
//If cancellation is requested at this point, just skip calling the retry policy and throw
//This is the most honest thing to do since we will not be issuing another request on the users behalf
//(since the cancellation token has already been set) and thus calling their retry policy would just
//be confusing.
!this.CancellationToken.IsCancellationRequested)
{
RequestInformation requestInformation;
if (batchException != null)
{
//If there is a BatchException, extract the RequestInformation and capture it
requestInformation = batchException.RequestInformation;
}
else
{
requestInformation = new RequestInformation()
{
ClientRequestId = this.Options.ClientRequestId
};
}
this.OperationContext.RequestResults.Add(new RequestResult(requestInformation, wrappedException)
{
Task = serviceRequestTask
});
capturedException = wrappedException;
}
else
{
if (batchException != null)
{
throw batchException; //Just forward the wrapped exception if there was one
}
else
{
throw;
}
}
}
if (capturedException != null)
{
//On an exception, invoke the retry policy
RetryDecision retryDecision = await this.RetryPolicy.ShouldRetryAsync(capturedException, this.OperationContext).ConfigureAwait(continueOnCapturedContext: false);
shouldRetry = retryDecision.ShouldRetry;
if (!shouldRetry)
{
//Rethrow the exception and explicitly preserve the stack trace
ExceptionDispatchInfo.Capture(capturedException).Throw();
}
else
{
if (retryDecision.RetryDelay.HasValue)
{
await Task.Delay(retryDecision.RetryDelay.Value).ConfigureAwait(continueOnCapturedContext: false);
}
else
{
Debug.Assert(false, "RetryDecision.ShouldRetry = true but RetryDelay has no value");
}
}
}
} while (shouldRetry);
//Reaching here is a bug, by now the request should have either thrown or returned
const string errorMessage = "Exited ExecuteRequestAsync without throwing or returning";
Debug.Assert(false, errorMessage);
throw new InvalidOperationException(errorMessage);
}
public RetryDecisionProviderStub(RetryDecision decision)
{
_decision = decision;
}
private static ConsistencyLevel CL(ConsistencyLevel cl, RetryDecision decision)
{
return(decision.RetryConsistencyLevel ?? cl);
}
/// <summary>
/// Defines whether to retry and at which consistency level on an unavailable
/// exception. <p> This method triggers a maximum of one retry. If at least one
/// replica is know to be alive, the operation is retried at a lower consistency
/// level.</p>
/// </summary>
/// <param name="query"> the original query for which the consistency level
/// cannot be achieved. </param>
/// <param name="cl"> the original consistency level for the operation. </param>
/// <param name="requiredReplica"> the number of replica that should have been
/// (known) alive for the operation to be attempted. </param>
/// <param name="aliveReplica"> the number of replica that were know to be alive
/// by the coordinator of the operation. </param>
/// <param name="nbRetry"> the number of retry already performed for this
/// operation. </param>
///
/// <returns>a RetryDecision as defined above.</returns>
public RetryDecision OnUnavailable(Query query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
return(nbRetry != 0 ? RetryDecision.Rethrow() : MaxLikelyToWorkCl(aliveReplica));
// Tries the biggest CL that is expected to work
}
public RetryDecision OnRequestError(IStatement statement, Configuration config, Exception ex, int nbRetry)
{
return(RetryDecision.Ignore());
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
return(RetryDecision.Retry(ConsistencyLevel.One, false));
}
public void After(int tries, RetryDecision action)
{
_rules.Add(tries, action);
}
public void Should_RetryRequestToSameHost_When_ConnectionFailsAndRetryDecisionIsRetrySameHost()
{
var mockSession = Mock.Of <IInternalSession>();
var config = new Configuration();
Mock.Get(mockSession).SetupGet(m => m.Cluster.Configuration).Returns(config);
var mockRequest = Mock.Of <IRequest>();
var mockParent = Mock.Of <IRequestHandler>();
var connection = Mock.Of <IConnection>();
// Setup hosts
var host = new Host(
new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9047),
new ConstantReconnectionPolicy(1));
var validHost = ValidHost.New(
host,
HostDistance.Local);
var secondHost = new Host(
new IPEndPoint(IPAddress.Parse("127.0.0.2"), 9047),
new ConstantReconnectionPolicy(1)); // second host should never be used if test passes
var secondValidHost = ValidHost.New(
secondHost,
HostDistance.Local);
// Setup query plan
Mock.Get(mockParent)
.SetupSequence(m => m.GetNextValidHost(It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.Returns(validHost)
.Returns(secondValidHost);
// Setup retry policy
var exception = new OverloadedException(string.Empty);
Mock.Get(mockParent)
.SetupGet(m => m.RetryPolicy)
.Returns(() =>
Mock.Of <IExtendedRetryPolicy>(a =>
a.OnRequestError(
It.IsAny <IStatement>(), config, exception, 0)
== RetryDecision.Retry(null, true)));
// Setup connection failure
Mock.Get(mockParent)
.Setup(m => m.GetConnectionToValidHostAsync(validHost, It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.ThrowsAsync(exception);
// Setup successful second connection on the same host retry (different method call - ValidateHostAndGetConnectionAsync)
Mock.Get(mockParent)
.Setup(m => m.ValidateHostAndGetConnectionAsync(validHost.Host, It.IsAny <Dictionary <IPEndPoint, Exception> >()))
.ReturnsAsync(connection);
var sut = new ProxyRequestExecution(mockParent, mockSession, mockRequest);
sut.Start(false);
// Validate request is sent
TestHelper.RetryAssert(
() =>
{
Mock.Get(connection).Verify(
c => c.Send(mockRequest, It.IsAny <Action <Exception, Response> >(), It.IsAny <int>()),
Times.Once);
});
// Validate that there were 2 connection attempts (1 with each method)
Mock.Get(mockParent).Verify(
m => m.GetConnectionToValidHostAsync(validHost, It.IsAny <Dictionary <IPEndPoint, Exception> >()),
Times.Once);
Mock.Get(mockParent).Verify(
m => m.ValidateHostAndGetConnectionAsync(validHost.Host, It.IsAny <Dictionary <IPEndPoint, Exception> >()),
Times.Once);
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved, int nbRetry)
{
Interlocked.Increment(ref ReadTimeoutCounter);
return(RetryDecision.Rethrow());
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
UnavailableCounter++;
return(RetryDecision.Ignore());
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Interlocked.Increment(ref WriteTimeoutCounter);
return(RetryDecision.Rethrow());
}
public RetryDecision OnWriteTimeout(Query query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
return(RetryDecision.Rethrow());
}
public RetryDecision OnUnavailable(IStatement query, ConsistencyLevel cl, int requiredReplica, int aliveReplica, int nbRetry)
{
Interlocked.Increment(ref UnavailableCounter);
return(RetryDecision.Rethrow());
}
public RetryDecision OnReadTimeout(IStatement query, ConsistencyLevel cl, int requiredResponses, int receivedResponses, bool dataRetrieved,
int nbRetry)
{
return(RetryDecision.Ignore());
}
public RetryDecision OnWriteTimeout(IStatement query, ConsistencyLevel cl, string writeType, int requiredAcks, int receivedAcks, int nbRetry)
{
Task.Delay(50).Wait();
return(RetryDecision.Retry(cl));
}
