Exemple #1
0
        private void RefreshAmbassadors(ClusterTopology clusterTopology, Dictionary <string, RemoteConnection> connections = null)
        {
            bool lockTaken = false;

            Monitor.TryEnter(this, ref lockTaken);
            try
            {
                //This only means we are been disposed so we can quit now
                if (lockTaken == false)
                {
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: Skipping refreshing ambassadors because we are been disposed of");
                    }
                    return;
                }

                if (Term != _engine.CurrentTerm)
                {
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: We are no longer the actual leader, since the current term is {_engine.CurrentTerm}");
                    }
                    return;
                }

                if (_engine.Log.IsInfoEnabled)
                {
                    _engine.Log.Info($"{ToString()}: Refreshing ambassadors");
                }
                var old = new Dictionary <string, FollowerAmbassador>(StringComparer.OrdinalIgnoreCase);
                foreach (var peers in new[] { _voters, _promotables, _nonVoters })
                {
                    foreach (var peer in peers)
                    {
                        old[peer.Key] = peer.Value;
                    }
                    peers.Clear();
                }

                foreach (var voter in clusterTopology.Members)
                {
                    if (voter.Key == _engine.Tag)
                    {
                        continue; // we obviously won't be applying to ourselves
                    }
                    if (old.TryGetValue(voter.Key, out FollowerAmbassador existingInstance))
                    {
                        existingInstance.UpdateLeaderWake(_voterResponded);
                        _voters[voter.Key] = existingInstance;
                        old.Remove(voter.Key);
                        continue; // already here
                    }
                    RemoteConnection connection = null;
                    connections?.TryGetValue(voter.Key, out connection);
                    var ambassador = new FollowerAmbassador(_engine, this, _voterResponded, voter.Key, voter.Value, connection);
                    _voters[voter.Key] = ambassador;
                    _engine.AppendStateDisposable(this, ambassador);
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: starting ambassador for voter {voter.Key} {voter.Value}");
                    }
                    ambassador.Start();
                }

                foreach (var promotable in clusterTopology.Promotables)
                {
                    if (old.TryGetValue(promotable.Key, out FollowerAmbassador existingInstance))
                    {
                        existingInstance.UpdateLeaderWake(_promotableUpdated);
                        _promotables[promotable.Key] = existingInstance;
                        old.Remove(promotable.Key);
                        continue; // already here
                    }
                    RemoteConnection connection = null;
                    connections?.TryGetValue(promotable.Key, out connection);
                    var ambassador = new FollowerAmbassador(_engine, this, _promotableUpdated, promotable.Key, promotable.Value, connection);
                    _promotables[promotable.Key] = ambassador;
                    _engine.AppendStateDisposable(this, ambassador);
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: starting ambassador for promotable {promotable.Key} {promotable.Value}");
                    }
                    ambassador.Start();
                }

                foreach (var nonVoter in clusterTopology.Watchers)
                {
                    if (old.TryGetValue(nonVoter.Key, out FollowerAmbassador existingInstance))
                    {
                        existingInstance.UpdateLeaderWake(_noop);

                        _nonVoters[nonVoter.Key] = existingInstance;
                        old.Remove(nonVoter.Key);
                        continue; // already here
                    }
                    RemoteConnection connection = null;
                    connections?.TryGetValue(nonVoter.Key, out connection);
                    var ambassador = new FollowerAmbassador(_engine, this, _noop, nonVoter.Key, nonVoter.Value, connection);
                    _nonVoters[nonVoter.Key] = ambassador;
                    _engine.AppendStateDisposable(this, ambassador);
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: starting ambassador for watcher {nonVoter.Key} {nonVoter.Value}");
                    }
                    ambassador.Start();
                }

                if (old.Count > 0)
                {
                    foreach (var ambassador in old)
                    {
                        _voters.TryRemove(ambassador.Key, out _);
                        _nonVoters.TryRemove(ambassador.Key, out _);
                        _promotables.TryRemove(ambassador.Key, out _);
                    }
                    Interlocked.Increment(ref _previousPeersWereDisposed);
                    System.Threading.ThreadPool.QueueUserWorkItem(_ =>
                    {
                        foreach (var ambassador in old)
                        {
                            // it is not used by anything else, so we can close it
                            ambassador.Value.Dispose();
                        }
                        Interlocked.Decrement(ref _previousPeersWereDisposed);
                    }, null);
                }
            }
            finally
            {
                if (lockTaken)
                {
                    Monitor.Exit(this);
                }
            }
        }
Exemple #2
0
        private void NegotiateMatchEntryWithLeaderAndApplyEntries(TransactionOperationContext context,
                                                                  RemoteConnection connection, LogLengthNegotiation negotiation)
        {
            long minIndex;
            long maxIndex;
            long midpointTerm;
            long midpointIndex;

            using (context.OpenReadTransaction())
            {
                minIndex = _engine.GetFirstEntryIndex(context);

                if (minIndex == 0) // no entries at all
                {
                    connection.Send(context, new LogLengthNegotiationResponse
                    {
                        Status       = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
                        Message      = "No entries at all here, give me everything from the start",
                        CurrentTerm  = _term,
                        LastLogIndex = 0,
                    });

                    return; // leader will know where to start from here
                }

                maxIndex = Math.Min(
                    _engine.GetLastEntryIndex(context), // max
                    negotiation.PrevLogIndex
                    );

                midpointIndex = (maxIndex + minIndex) / 2;

                midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex);
            }

            while ((midpointTerm == negotiation.PrevLogTerm && midpointIndex == negotiation.PrevLogIndex) == false)
            {
                _engine.Timeout.Defer(_connection.Source);

                _engine.ValidateTerm(_term);

                if (midpointIndex == negotiation.PrevLogIndex && midpointTerm != negotiation.PrevLogTerm)
                {
                    // our appended entries has been diverged, same index with different terms.
                    var msg = $"Our appended entries has been diverged, same index with different terms. " +
                              $"My index/term {midpointIndex}/{midpointTerm}, while yours is {negotiation.PrevLogIndex}/{negotiation.PrevLogTerm}.";
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: {msg}");
                    }
                    connection.Send(context, new LogLengthNegotiationResponse
                    {
                        Status       = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
                        Message      = msg,
                        CurrentTerm  = _term,
                        LastLogIndex = 0
                    });
                    return;
                }

                connection.Send(context, new LogLengthNegotiationResponse
                {
                    Status  = LogLengthNegotiationResponse.ResponseStatus.Negotiation,
                    Message =
                        $"Term/Index mismatch from leader, need to figure out at what point the logs match, range: {maxIndex} - {minIndex} | {midpointIndex} in term {midpointTerm}",
                    CurrentTerm   = _term,
                    MaxIndex      = maxIndex,
                    MinIndex      = minIndex,
                    MidpointIndex = midpointIndex,
                    MidpointTerm  = midpointTerm
                });

                negotiation = connection.Read <LogLengthNegotiation>(context);

                _engine.Timeout.Defer(_connection.Source);
                if (negotiation.Truncated)
                {
                    if (_engine.Log.IsInfoEnabled)
                    {
                        _engine.Log.Info($"{ToString()}: Got a truncated response from the leader will request all entries");
                    }
                    connection.Send(context, new LogLengthNegotiationResponse
                    {
                        Status       = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
                        Message      = "We have entries that are already truncated at the leader, will ask for full snapshot",
                        CurrentTerm  = _term,
                        LastLogIndex = 0
                    });
                    return;
                }
                using (context.OpenReadTransaction())
                {
                    if (_engine.GetTermFor(context, negotiation.PrevLogIndex) == negotiation.PrevLogTerm)
                    {
                        minIndex = Math.Min(midpointIndex + 1, maxIndex);
                    }
                    else
                    {
                        maxIndex = Math.Max(midpointIndex - 1, minIndex);
                    }
                }
                midpointIndex = (maxIndex + minIndex) / 2;
                using (context.OpenReadTransaction())
                    midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex);
            }

            if (_engine.Log.IsInfoEnabled)
            {
                _engine.Log.Info($"{ToString()}: agreed upon last matched index = {midpointIndex} on term = {midpointTerm}");
            }

            connection.Send(context, new LogLengthNegotiationResponse
            {
                Status       = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
                Message      = $"Found a log index / term match at {midpointIndex} with term {midpointTerm}",
                CurrentTerm  = _term,
                LastLogIndex = midpointIndex,
            });
        }
Exemple #3
0
 public Reader(RemoteConnection parent)
 {
     _parent = parent;
 }
Exemple #4
0
 public Elector(RachisConsensus engine, RemoteConnection connection)
 {
     _engine     = engine;
     _connection = connection;
 }