Example #1
0
        private Task <T> ExecuteWithReplicationAsync <T>(ExecuteWithReplicationState <T> state)
        {
            switch (state.State)
            {
            case ExecuteWithReplicationStates.Start:
                state.ReplicationDestinations = ReplicationDestinationsUrls;

                var shouldReadFromAllServers = conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
                if (shouldReadFromAllServers && state.Method == "GET")
                {
                    var replicationIndex = state.ReadStripingBase % (state.ReplicationDestinations.Count + 1);
                    // if replicationIndex == destinations count, then we want to use the master
                    // if replicationIndex < 0, then we were explicitly instructed to use the master
                    if (replicationIndex < state.ReplicationDestinations.Count && replicationIndex >= 0)
                    {
                        // if it is failing, ignore that, and move to the master or any of the replicas
                        if (ShouldExecuteUsing(state.ReplicationDestinations[replicationIndex], state.CurrentRequest, state.Method, false))
                        {
                            return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.ReplicationDestinations[replicationIndex],
                                                                                          state.With(ExecuteWithReplicationStates.AfterTryingWithStripedServer),
                                                                                          state.ReplicationDestinations.Count > state.LastAttempt + 1));
                        }
                    }
                }

                goto case ExecuteWithReplicationStates.AfterTryingWithStripedServer;

            case ExecuteWithReplicationStates.AfterTryingWithStripedServer:

                if (!ShouldExecuteUsing(state.PrimaryUrl, state.CurrentRequest, state.Method, true))
                {
                    goto case ExecuteWithReplicationStates.TryAllServers;                             // skips both checks
                }
                return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.PrimaryUrl,
                                                                              state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrl),
                                                                              state.ReplicationDestinations.Count > state.LastAttempt + 1));

            case ExecuteWithReplicationStates.AfterTryingWithDefaultUrl:
                if (IsFirstFailure(state.PrimaryUrl))
                {
                    return(AttemptOperationAndOnFailureCallExecuteWithReplication(state.PrimaryUrl,
                                                                                  state.With(ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice),
                                                                                  state.ReplicationDestinations.Count > state.LastAttempt + 1));
                }

                goto case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice;

            case ExecuteWithReplicationStates.AfterTryingWithDefaultUrlTwice:

                IncrementFailureCount(state.PrimaryUrl);

                goto case ExecuteWithReplicationStates.TryAllServers;

            case ExecuteWithReplicationStates.TryAllServers:

                // The following part (cases ExecuteWithReplicationStates.TryAllServers, and ExecuteWithReplicationStates.TryAllServersSecondAttempt)
                // is a for loop, rolled out using goto and nested calls of the method in continuations
                state.LastAttempt++;
                if (state.LastAttempt >= state.ReplicationDestinations.Count)
                {
                    goto case ExecuteWithReplicationStates.AfterTryingAllServers;
                }

                var destination = state.ReplicationDestinations[state.LastAttempt];
                if (!ShouldExecuteUsing(destination, state.CurrentRequest, state.Method, false))
                {
                    // continue the next iteration of the loop
                    goto case ExecuteWithReplicationStates.TryAllServers;
                }

                return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination,
                                                                              state.With(ExecuteWithReplicationStates.TryAllServersSecondAttempt),
                                                                              state.ReplicationDestinations.Count > state.LastAttempt + 1));

            case ExecuteWithReplicationStates.TryAllServersSecondAttempt:
                destination = state.ReplicationDestinations[state.LastAttempt];
                if (IsFirstFailure(destination))
                {
                    return(AttemptOperationAndOnFailureCallExecuteWithReplication(destination,
                                                                                  state.With(ExecuteWithReplicationStates.TryAllServersFailedTwice),
                                                                                  state.ReplicationDestinations.Count > state.LastAttempt + 1));
                }

                goto case ExecuteWithReplicationStates.TryAllServersFailedTwice;

            case ExecuteWithReplicationStates.TryAllServersFailedTwice:
                IncrementFailureCount(state.ReplicationDestinations[state.LastAttempt]);

                // continue the next iteration of the loop
                goto case ExecuteWithReplicationStates.TryAllServers;

            case ExecuteWithReplicationStates.AfterTryingAllServers:
                throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + state.ReplicationDestinations.Count) + " Raven instances.");

            default:
                throw new InvalidOperationException("Invalid ExecuteWithReplicationState " + state);
            }
        }
Example #2
0
        protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(string url, ExecuteWithReplicationState <T> state, bool avoidThrowing)
        {
            Task <Task <T> > finalTask = state.Operation(url).ContinueWith(task =>
            {
                switch (task.Status)
                {
                case TaskStatus.RanToCompletion:
                    ResetFailureCount(url);
                    var tcs = new TaskCompletionSource <T>();
                    tcs.SetResult(task.Result);
                    return(tcs.Task);

                case TaskStatus.Canceled:
                    tcs = new TaskCompletionSource <T>();
                    tcs.SetCanceled();
                    return(tcs.Task);

                case TaskStatus.Faulted:
                    Debug.Assert(task.Exception != null);
                    if (IsServerDown(task.Exception) && avoidThrowing)
                    {
                        return(ExecuteWithReplicationAsync(state));
                    }

                    tcs = new TaskCompletionSource <T>();
                    tcs.SetException(task.Exception);
                    return(tcs.Task);

                default:
                    throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication");
                }
            });

            return(finalTask.Unwrap());
        }
Example #3
0
        protected virtual Task <T> AttemptOperationAndOnFailureCallExecuteWithReplication <T>(OperationMetadata operationMetadata, OperationMetadata primaryOperationMetadata, ExecuteWithReplicationState <T> state, bool avoidThrowing)
        {
            var tryWithPrimaryCredentials = IsFirstFailure(operationMetadata.Url) && primaryOperationMetadata != null;

            Task <Task <T> > finalTask = state.Operation(tryWithPrimaryCredentials ? new OperationMetadata(operationMetadata.Url, primaryOperationMetadata.Credentials) : operationMetadata).ContinueWith(task =>
            {
                switch (task.Status)
                {
                case TaskStatus.RanToCompletion:
                    ResetFailureCount(operationMetadata.Url);
                    var tcs = new TaskCompletionSource <T>();
                    tcs.SetResult(task.Result);
                    return(tcs.Task);

                case TaskStatus.Canceled:
                    tcs = new TaskCompletionSource <T>();
                    tcs.SetCanceled();
                    return(tcs.Task);

                case TaskStatus.Faulted:
                    Debug.Assert(task.Exception != null);

                    if (task.Exception != null)
                    {
                        var aggregateException = task.Exception;
                        var webException       = aggregateException.ExtractSingleInnerException() as WebException;

                        if (tryWithPrimaryCredentials && operationMetadata.Credentials.HasCredentials() && webException != null)
                        {
                            IncrementFailureCount(operationMetadata.Url);

                            var response = webException.Response as HttpWebResponse;
                            if (response != null && response.StatusCode == HttpStatusCode.Unauthorized)
                            {
                                return(AttemptOperationAndOnFailureCallExecuteWithReplication(operationMetadata, primaryOperationMetadata, state, avoidThrowing));
                            }
                        }
                    }

                    bool timeoutThrown;
                    if (IsServerDown(task.Exception, out timeoutThrown) && avoidThrowing)
                    {
                        state.TimeoutThrown = timeoutThrown;
                        return(ExecuteWithReplicationAsync(state));
                    }

                    tcs = new TaskCompletionSource <T>();
                    tcs.SetException(task.Exception);
                    return(tcs.Task);

                default:
                    throw new InvalidOperationException("Unknown task status in AttemptOperationAndOnFailureCallExecuteWithReplication");
                }
            });

            return(finalTask.Unwrap());
        }