private void RefreshAmbassadors(ClusterTopology clusterTopology, Dictionary <string, RemoteConnection> connections = null) { bool lockTaken = false; Monitor.TryEnter(this, ref lockTaken); try { //This only means we are been disposed so we can quit now if (lockTaken == false) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: Skipping refreshing ambassadors because we are been disposed of"); } return; } if (Term != _engine.CurrentTerm) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: We are no longer the actual leader, since the current term is {_engine.CurrentTerm}"); } return; } if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: Refreshing ambassadors"); } var old = new Dictionary <string, FollowerAmbassador>(StringComparer.OrdinalIgnoreCase); foreach (var peers in new[] { _voters, _promotables, _nonVoters }) { foreach (var peer in peers) { old[peer.Key] = peer.Value; } peers.Clear(); } foreach (var voter in clusterTopology.Members) { if (voter.Key == _engine.Tag) { continue; // we obviously won't be applying to ourselves } if (old.TryGetValue(voter.Key, out FollowerAmbassador existingInstance)) { existingInstance.UpdateLeaderWake(_voterResponded); _voters[voter.Key] = existingInstance; old.Remove(voter.Key); continue; // already here } RemoteConnection connection = null; connections?.TryGetValue(voter.Key, out connection); var ambassador = new FollowerAmbassador(_engine, this, _voterResponded, voter.Key, voter.Value, connection); _voters[voter.Key] = ambassador; _engine.AppendStateDisposable(this, ambassador); if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: starting ambassador for voter {voter.Key} {voter.Value}"); } ambassador.Start(); } foreach (var promotable in clusterTopology.Promotables) { if (old.TryGetValue(promotable.Key, out FollowerAmbassador existingInstance)) { existingInstance.UpdateLeaderWake(_promotableUpdated); _promotables[promotable.Key] = existingInstance; old.Remove(promotable.Key); continue; // already here } RemoteConnection connection = null; connections?.TryGetValue(promotable.Key, out connection); var ambassador = new FollowerAmbassador(_engine, this, _promotableUpdated, promotable.Key, promotable.Value, connection); _promotables[promotable.Key] = ambassador; _engine.AppendStateDisposable(this, ambassador); if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: starting ambassador for promotable {promotable.Key} {promotable.Value}"); } ambassador.Start(); } foreach (var nonVoter in clusterTopology.Watchers) { if (old.TryGetValue(nonVoter.Key, out FollowerAmbassador existingInstance)) { existingInstance.UpdateLeaderWake(_noop); _nonVoters[nonVoter.Key] = existingInstance; old.Remove(nonVoter.Key); continue; // already here } RemoteConnection connection = null; connections?.TryGetValue(nonVoter.Key, out connection); var ambassador = new FollowerAmbassador(_engine, this, _noop, nonVoter.Key, nonVoter.Value, connection); _nonVoters[nonVoter.Key] = ambassador; _engine.AppendStateDisposable(this, ambassador); if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: starting ambassador for watcher {nonVoter.Key} {nonVoter.Value}"); } ambassador.Start(); } if (old.Count > 0) { foreach (var ambassador in old) { _voters.TryRemove(ambassador.Key, out _); _nonVoters.TryRemove(ambassador.Key, out _); _promotables.TryRemove(ambassador.Key, out _); } Interlocked.Increment(ref _previousPeersWereDisposed); System.Threading.ThreadPool.QueueUserWorkItem(_ => { foreach (var ambassador in old) { // it is not used by anything else, so we can close it ambassador.Value.Dispose(); } Interlocked.Decrement(ref _previousPeersWereDisposed); }, null); } } finally { if (lockTaken) { Monitor.Exit(this); } } }
private void NegotiateMatchEntryWithLeaderAndApplyEntries(TransactionOperationContext context, RemoteConnection connection, LogLengthNegotiation negotiation) { long minIndex; long maxIndex; long midpointTerm; long midpointIndex; using (context.OpenReadTransaction()) { minIndex = _engine.GetFirstEntryIndex(context); if (minIndex == 0) // no entries at all { connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable, Message = "No entries at all here, give me everything from the start", CurrentTerm = _term, LastLogIndex = 0, }); return; // leader will know where to start from here } maxIndex = Math.Min( _engine.GetLastEntryIndex(context), // max negotiation.PrevLogIndex ); midpointIndex = (maxIndex + minIndex) / 2; midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex); } while ((midpointTerm == negotiation.PrevLogTerm && midpointIndex == negotiation.PrevLogIndex) == false) { _engine.Timeout.Defer(_connection.Source); _engine.ValidateTerm(_term); if (midpointIndex == negotiation.PrevLogIndex && midpointTerm != negotiation.PrevLogTerm) { // our appended entries has been diverged, same index with different terms. var msg = $"Our appended entries has been diverged, same index with different terms. " + $"My index/term {midpointIndex}/{midpointTerm}, while yours is {negotiation.PrevLogIndex}/{negotiation.PrevLogTerm}."; if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: {msg}"); } connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable, Message = msg, CurrentTerm = _term, LastLogIndex = 0 }); return; } connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Negotiation, Message = $"Term/Index mismatch from leader, need to figure out at what point the logs match, range: {maxIndex} - {minIndex} | {midpointIndex} in term {midpointTerm}", CurrentTerm = _term, MaxIndex = maxIndex, MinIndex = minIndex, MidpointIndex = midpointIndex, MidpointTerm = midpointTerm }); negotiation = connection.Read <LogLengthNegotiation>(context); _engine.Timeout.Defer(_connection.Source); if (negotiation.Truncated) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: Got a truncated response from the leader will request all entries"); } connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable, Message = "We have entries that are already truncated at the leader, will ask for full snapshot", CurrentTerm = _term, LastLogIndex = 0 }); return; } using (context.OpenReadTransaction()) { if (_engine.GetTermFor(context, negotiation.PrevLogIndex) == negotiation.PrevLogTerm) { minIndex = Math.Min(midpointIndex + 1, maxIndex); } else { maxIndex = Math.Max(midpointIndex - 1, minIndex); } } midpointIndex = (maxIndex + minIndex) / 2; using (context.OpenReadTransaction()) midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex); } if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: agreed upon last matched index = {midpointIndex} on term = {midpointTerm}"); } connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable, Message = $"Found a log index / term match at {midpointIndex} with term {midpointTerm}", CurrentTerm = _term, LastLogIndex = midpointIndex, }); }
public Reader(RemoteConnection parent) { _parent = parent; }
public Elector(RachisConsensus engine, RemoteConnection connection) { _engine = engine; _connection = connection; }