private void RefreshAmbassadors(ClusterTopology clusterTopology, Dictionary <string, RemoteConnection> connections = null)
{
bool lockTaken = false;
Monitor.TryEnter(this, ref lockTaken);
try
{
//This only means we are been disposed so we can quit now
if (lockTaken == false)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Skipping refreshing ambassadors because we are been disposed of");
}
return;
}
if (Term != _engine.CurrentTerm)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: We are no longer the actual leader, since the current term is {_engine.CurrentTerm}");
}
return;
}
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Refreshing ambassadors");
}
var old = new Dictionary <string, FollowerAmbassador>(StringComparer.OrdinalIgnoreCase);
foreach (var peers in new[] { _voters, _promotables, _nonVoters })
{
foreach (var peer in peers)
{
old[peer.Key] = peer.Value;
}
peers.Clear();
}
foreach (var voter in clusterTopology.Members)
{
if (voter.Key == _engine.Tag)
{
continue; // we obviously won't be applying to ourselves
}
if (old.TryGetValue(voter.Key, out FollowerAmbassador existingInstance))
{
existingInstance.UpdateLeaderWake(_voterResponded);
_voters[voter.Key] = existingInstance;
old.Remove(voter.Key);
continue; // already here
}
RemoteConnection connection = null;
connections?.TryGetValue(voter.Key, out connection);
var ambassador = new FollowerAmbassador(_engine, this, _voterResponded, voter.Key, voter.Value, connection);
_voters[voter.Key] = ambassador;
_engine.AppendStateDisposable(this, ambassador);
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: starting ambassador for voter {voter.Key} {voter.Value}");
}
ambassador.Start();
}
foreach (var promotable in clusterTopology.Promotables)
{
if (old.TryGetValue(promotable.Key, out FollowerAmbassador existingInstance))
{
existingInstance.UpdateLeaderWake(_promotableUpdated);
_promotables[promotable.Key] = existingInstance;
old.Remove(promotable.Key);
continue; // already here
}
RemoteConnection connection = null;
connections?.TryGetValue(promotable.Key, out connection);
var ambassador = new FollowerAmbassador(_engine, this, _promotableUpdated, promotable.Key, promotable.Value, connection);
_promotables[promotable.Key] = ambassador;
_engine.AppendStateDisposable(this, ambassador);
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: starting ambassador for promotable {promotable.Key} {promotable.Value}");
}
ambassador.Start();
}
foreach (var nonVoter in clusterTopology.Watchers)
{
if (old.TryGetValue(nonVoter.Key, out FollowerAmbassador existingInstance))
{
existingInstance.UpdateLeaderWake(_noop);
_nonVoters[nonVoter.Key] = existingInstance;
old.Remove(nonVoter.Key);
continue; // already here
}
RemoteConnection connection = null;
connections?.TryGetValue(nonVoter.Key, out connection);
var ambassador = new FollowerAmbassador(_engine, this, _noop, nonVoter.Key, nonVoter.Value, connection);
_nonVoters[nonVoter.Key] = ambassador;
_engine.AppendStateDisposable(this, ambassador);
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: starting ambassador for watcher {nonVoter.Key} {nonVoter.Value}");
}
ambassador.Start();
}
if (old.Count > 0)
{
foreach (var ambassador in old)
{
_voters.TryRemove(ambassador.Key, out _);
_nonVoters.TryRemove(ambassador.Key, out _);
_promotables.TryRemove(ambassador.Key, out _);
}
Interlocked.Increment(ref _previousPeersWereDisposed);
System.Threading.ThreadPool.QueueUserWorkItem(_ =>
{
foreach (var ambassador in old)
{
// it is not used by anything else, so we can close it
ambassador.Value.Dispose();
}
Interlocked.Decrement(ref _previousPeersWereDisposed);
}, null);
}
}
finally
{
if (lockTaken)
{
Monitor.Exit(this);
}
}
}
private void NegotiateMatchEntryWithLeaderAndApplyEntries(TransactionOperationContext context,
RemoteConnection connection, LogLengthNegotiation negotiation)
{
long minIndex;
long maxIndex;
long midpointTerm;
long midpointIndex;
using (context.OpenReadTransaction())
{
minIndex = _engine.GetFirstEntryIndex(context);
if (minIndex == 0) // no entries at all
{
connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = "No entries at all here, give me everything from the start",
CurrentTerm = _term,
LastLogIndex = 0,
});
return; // leader will know where to start from here
}
maxIndex = Math.Min(
_engine.GetLastEntryIndex(context), // max
negotiation.PrevLogIndex
);
midpointIndex = (maxIndex + minIndex) / 2;
midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex);
}
while ((midpointTerm == negotiation.PrevLogTerm && midpointIndex == negotiation.PrevLogIndex) == false)
{
_engine.Timeout.Defer(_connection.Source);
_engine.ValidateTerm(_term);
if (midpointIndex == negotiation.PrevLogIndex && midpointTerm != negotiation.PrevLogTerm)
{
// our appended entries has been diverged, same index with different terms.
var msg = $"Our appended entries has been diverged, same index with different terms. " +
$"My index/term {midpointIndex}/{midpointTerm}, while yours is {negotiation.PrevLogIndex}/{negotiation.PrevLogTerm}.";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: {msg}");
}
connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = msg,
CurrentTerm = _term,
LastLogIndex = 0
});
return;
}
connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Negotiation,
Message =
$"Term/Index mismatch from leader, need to figure out at what point the logs match, range: {maxIndex} - {minIndex} | {midpointIndex} in term {midpointTerm}",
CurrentTerm = _term,
MaxIndex = maxIndex,
MinIndex = minIndex,
MidpointIndex = midpointIndex,
MidpointTerm = midpointTerm
});
negotiation = connection.Read <LogLengthNegotiation>(context);
_engine.Timeout.Defer(_connection.Source);
if (negotiation.Truncated)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a truncated response from the leader will request all entries");
}
connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = "We have entries that are already truncated at the leader, will ask for full snapshot",
CurrentTerm = _term,
LastLogIndex = 0
});
return;
}
using (context.OpenReadTransaction())
{
if (_engine.GetTermFor(context, negotiation.PrevLogIndex) == negotiation.PrevLogTerm)
{
minIndex = Math.Min(midpointIndex + 1, maxIndex);
}
else
{
maxIndex = Math.Max(midpointIndex - 1, minIndex);
}
}
midpointIndex = (maxIndex + minIndex) / 2;
using (context.OpenReadTransaction())
midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex);
}
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: agreed upon last matched index = {midpointIndex} on term = {midpointTerm}");
}
connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = $"Found a log index / term match at {midpointIndex} with term {midpointTerm}",
CurrentTerm = _term,
LastLogIndex = midpointIndex,
});
}
public Reader(RemoteConnection parent)
{
_parent = parent;
}
public Elector(RachisConsensus engine, RemoteConnection connection)
{
_engine = engine;
_connection = connection;
}