private Task <T> ExecuteWithReplicationAsync <T>(ExecuteWithReplicationState <T> state)
{
switch (state.State)
{
case ExecuteWithReplicationStates.Start:
state.ReplicationDestinations = ReplicationDestinationsUrls;
var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
if (shouldReadFromAllServers && state.Method == "GET")
{
var replicationIndex = state.ReadStripingBase % (state.ReplicationDestinations.Count + 1);
// if replicationIndex == destinations count, then we want to use the master
// if replicationIndex < 0, then we were explicitly instructed to use the master
if (replicationIndex < state.ReplicationDestinations.Count && replicationIndex >= 0)
{
// if it is failing, ignore that, and move to the master or any of the replicas
if (ShouldExecuteUsing(state.ReplicationDestinations[replicationIndex], state.CurrentRequest, state.Method, false))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.ReplicationDestinations[replicationIndex],
state.With(ExecuteWithReplicationStates.AfterTryingWithStripedServer),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
}
}
goto case ExecuteWithReplicationStates.AfterTryingWithStripedServer;
case ExecuteWithReplicationStates.AfterTryingWithStripedServer:
if (!ShouldExecuteUsing(state.PrimaryUrl, state.CurrentRequest, state.Method, true))
{
goto case ExecuteWithReplicationStates.TryAllServers; // skips both checks
}
return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.PrimaryUrl,
state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrl),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
case ExecuteWithReplicationStates.AfterTryingWithDefaultUrl:
if (IsFirstFailure(state.PrimaryUrl))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.PrimaryUrl,
state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
goto case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice;
case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice:
IncrementFailureCount(state.PrimaryUrl);
goto case ExecuteWithReplicationStates.TryAllServers;
case ExecuteWithReplicationStates.TryAllServers:
// The following part (cases ExecuteWithReplicationStates.TryAllServers, and ExecuteWithReplicationStates.TryAllServersSecondAttempt)
// is a for loop, rolled out using goto and nested calls of the method in continuations
state.LastAttempt++;
if (state.LastAttempt >= state.ReplicationDestinations.Count)
{
goto case ExecuteWithReplicationStates.AfterTryingAllServers;
}
var destination = state.ReplicationDestinations[state.LastAttempt];
if (!ShouldExecuteUsing(destination, state.CurrentRequest, state.Method, false))
{
// continue the next iteration of the loop
goto case ExecuteWithReplicationStates.TryAllServers;
}
return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination,
state.With(ExecuteWithReplicationStates.TryAllServersSecondAttempt),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
case ExecuteWithReplicationStates.TryAllServersSecondAttempt:
destination = state.ReplicationDestinations[state.LastAttempt];
if (IsFirstFailure(destination))
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination,
state.With(ExecuteWithReplicationStates.TryAllServersFailedTwice),
state.ReplicationDestinations.Count > state.LastAttempt + 1));
}
goto case ExecuteWithReplicationStates.TryAllServersFailedTwice;
case ExecuteWithReplicationStates.TryAllServersFailedTwice:
IncrementFailureCount(state.ReplicationDestinations[state.LastAttempt]);
// continue the next iteration of the loop
goto case ExecuteWithReplicationStates.TryAllServers;
case ExecuteWithReplicationStates.AfterTryingAllServers:
throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + state.ReplicationDestinations.Count) + " Raven instances.");
default:
throw new InvalidOperationException("Invalid ExecuteWithReplicationState " + state);
}
}
protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(string url, ExecuteWithReplicationState <T> state, bool avoidThrowing)
{
Task <Task <T> > finalTask = state.Operation(url).ContinueWith(task =>
{
switch (task.Status)
{
case TaskStatus.RanToCompletion:
ResetFailureCount(url);
var tcs = new TaskCompletionSource <T>();
tcs.SetResult(task.Result);
return(tcs.Task);
case TaskStatus.Canceled:
tcs = new TaskCompletionSource <T>();
tcs.SetCanceled();
return(tcs.Task);
case TaskStatus.Faulted:
Debug.Assert(task.Exception != null);
if (IsServerDown(task.Exception) && avoidThrowing)
{
return(ExecuteWithReplicationAsync(state));
}
tcs = new TaskCompletionSource <T>();
tcs.SetException(task.Exception);
return(tcs.Task);
default:
throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication");
}
});
return(finalTask.Unwrap());
}
protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, ExecuteWithReplicationState <T> state, bool avoidThrowing)
{
var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;
Task <Task <T> > finalTask = state.Operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ContinueWith(task =>
{
switch (task.Status)
{
case TaskStatus.RanToCompletion:
ResetFailureCount(operationMetadata.Url);
var tcs = new TaskCompletionSource <T>();
tcs.SetResult(task.Result);
return(tcs.Task);
case TaskStatus.Canceled:
tcs = new TaskCompletionSource <T>();
tcs.SetCanceled();
return(tcs.Task);
case TaskStatus.Faulted:
Debug.Assert(task.Exception != null);
if (task.Exception != null)
{
var aggregateException = task.Exception;
var webException = aggregateException.ExtractSingleInnerException() as WebException;
if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null)
{
IncrementFailureCount(operationMetadata.Url);
var response = webException.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.Unauthorized)
{
return(AttemptOperationAndOnFailureCallExecuteWithReplication(operationMetadata, primaryOperationMetadata, state, avoidThrowing));
}
}
}
bool timeoutThrown;
if (IsServerDown(task.Exception, out timeoutThrown) && avoidThrowing)
{
state.TimeoutThrown = timeoutThrown;
return(ExecuteWithReplicationAsync(state));
}
tcs = new TaskCompletionSource <T>();
tcs.SetException(task.Exception);
return(tcs.Task);
default:
throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication");
}
});
return(finalTask.Unwrap());
}
