private void SendProposal()
{
_acceptsReceived.Clear();
_leaderProposal = null;
var leader = GetBestLeaderCandidate(_prepareOkReceived, _servers, _resigningLeaderInstanceId, _lastAttemptedView);
if (leader == null)
{
Log.Information("ELECTIONS: (V={lastAttemptedView}) NO LEADER CANDIDATE WHEN TRYING TO SEND PROPOSAL.",
_lastAttemptedView);
return;
}
_leaderProposal = leader;
Log.Information("ELECTIONS: (V={lastAttemptedView}) SENDING PROPOSAL CANDIDATE: {formatNodeInfo}, ME: {ownInfo}.",
_lastAttemptedView, FormatNodeInfo(leader), FormatNodeInfo(GetOwnInfo()));
var proposal = new ElectionMessage.Proposal(_memberInfo.InstanceId, _memberInfo.HttpEndPoint,
leader.InstanceId, leader.HttpEndPoint,
_lastInstalledView,
leader.EpochNumber, leader.EpochPosition, leader.EpochId, leader.EpochLeaderInstanceId,
leader.LastCommitPosition, leader.WriterCheckpoint, leader.ChaserCheckpoint, leader.NodePriority);
Handle(new ElectionMessage.Accept(_memberInfo.InstanceId, _memberInfo.HttpEndPoint,
leader.InstanceId, leader.HttpEndPoint, _lastInstalledView));
SendToAllExceptMe(proposal);
}
private static bool IsCandidateGoodEnough(LeaderCandidate candidate, LeaderCandidate ownInfo)
{
if (candidate.EpochNumber != ownInfo.EpochNumber)
{
return(candidate.EpochNumber > ownInfo.EpochNumber);
}
if (candidate.LastCommitPosition != ownInfo.LastCommitPosition)
{
return(candidate.LastCommitPosition > ownInfo.LastCommitPosition);
}
if (candidate.WriterCheckpoint != ownInfo.WriterCheckpoint)
{
return(candidate.WriterCheckpoint > ownInfo.WriterCheckpoint);
}
if (candidate.ChaserCheckpoint != ownInfo.ChaserCheckpoint)
{
return(candidate.ChaserCheckpoint > ownInfo.ChaserCheckpoint);
}
return(true);
}
public static bool IsLegitimateLeader(int view, EndPoint proposingServerEndPoint, Guid proposingServerId,
LeaderCandidate candidate, MemberInfo[] servers, Guid?lastElectedLeader, Guid instanceId,
LeaderCandidate ownInfo,
Guid?resigningLeader)
{
var leader = servers.FirstOrDefault(x =>
x.IsAlive && x.InstanceId == lastElectedLeader && x.State == VNodeState.Leader);
if (leader != null && leader.InstanceId != resigningLeader)
{
if (candidate.InstanceId == leader.InstanceId ||
candidate.EpochNumber > leader.EpochNumber ||
(candidate.EpochNumber == leader.EpochNumber && candidate.EpochId != leader.EpochId))
{
return(true);
}
Log.Information(
"ELECTIONS: (V={view}) NOT LEGITIMATE LEADER PROPOSAL FROM [{proposingServerEndPoint},{proposingServerId:B}] M={candidateInfo}. "
+ "PREVIOUS LEADER IS ALIVE: [{leaderHttpEndPoint},{leaderId:B}].",
view, proposingServerEndPoint, proposingServerId, FormatNodeInfo(candidate),
leader.HttpEndPoint, leader.InstanceId);
return(false);
}
if (candidate.InstanceId == instanceId)
{
return(true);
}
if (!IsCandidateGoodEnough(candidate, ownInfo))
{
Log.Information(
"ELECTIONS: (V={view}) NOT LEGITIMATE LEADER PROPOSAL FROM [{proposingServerEndPoint},{proposingServerId:B}] M={candidateInfo}. ME={ownInfo}.",
view, proposingServerEndPoint, proposingServerId, FormatNodeInfo(candidate),
FormatNodeInfo(ownInfo));
return(false);
}
return(true);
}
private void ShiftToLeaderElection(int view)
{
Log.Information("ELECTIONS: (V={view}) SHIFT TO LEADER ELECTION.", view);
_state = ElectionsState.ElectingLeader;
_vcReceived.Clear();
_prepareOkReceived.Clear();
_lastAttemptedView = view;
_leaderProposal = null;
_leader = null;
_acceptsReceived.Clear();
var viewChangeMsg = new ElectionMessage.ViewChange(_memberInfo.InstanceId, _memberInfo.HttpEndPoint, view);
Handle(viewChangeMsg);
SendToAllExceptMe(viewChangeMsg);
_publisher.Publish(TimerMessage.Schedule.Create(LeaderElectionProgressTimeout,
_publisherEnvelope,
new ElectionMessage.ElectionsTimedOut(view)));
}
private static string FormatNodeInfo(LeaderCandidate candidate)
{
return(FormatNodeInfo(candidate.HttpEndPoint, candidate.InstanceId,
candidate.LastCommitPosition, candidate.WriterCheckpoint, candidate.ChaserCheckpoint,
candidate.EpochNumber, candidate.EpochPosition, candidate.EpochId, candidate.EpochLeaderInstanceId, candidate.NodePriority));
}
public void Handle(ElectionMessage.Proposal message)
{
if (_state == ElectionsState.Shutdown)
{
return;
}
if (message.ServerId == _memberInfo.InstanceId)
{
return;
}
if (_state != ElectionsState.Acceptor)
{
return;
}
if (message.View != _lastInstalledView)
{
return;
}
if (_servers.All(x => x.InstanceId != message.ServerId))
{
return;
}
if (_servers.All(x => x.InstanceId != message.LeaderId))
{
return;
}
var candidate = new LeaderCandidate(message.LeaderId, message.LeaderHttpEndPoint,
message.EpochNumber, message.EpochPosition, message.EpochId, message.EpochLeaderInstanceId,
message.LastCommitPosition, message.WriterCheckpoint, message.ChaserCheckpoint, message.NodePriority);
var ownInfo = GetOwnInfo();
if (!IsLegitimateLeader(message.View, message.ServerHttpEndPoint, message.ServerId,
candidate, _servers, _lastElectedLeader, _memberInfo.InstanceId, ownInfo,
_resigningLeaderInstanceId))
{
return;
}
Log.Information(
"ELECTIONS: (V={lastAttemptedView}) PROPOSAL FROM [{serverHttpEndPoint},{serverId:B}] M={candidateInfo}. ME={ownInfo}, NodePriority={priority}",
_lastAttemptedView,
message.ServerHttpEndPoint, message.ServerId, FormatNodeInfo(candidate), FormatNodeInfo(GetOwnInfo()),
message.NodePriority);
if (_leaderProposal == null)
{
_leaderProposal = candidate;
_acceptsReceived.Clear();
}
if (_leaderProposal.InstanceId == message.LeaderId)
{
// NOTE: proposal from other server is also implicit Accept from that server
Handle(new ElectionMessage.Accept(message.ServerId, message.ServerHttpEndPoint,
message.LeaderId, message.LeaderHttpEndPoint, message.View));
var accept = new ElectionMessage.Accept(_memberInfo.InstanceId, _memberInfo.HttpEndPoint,
message.LeaderId, message.LeaderHttpEndPoint, message.View);
Handle(accept); // implicitly sent accept to ourselves
SendToAllExceptMe(accept);
}
}
public static LeaderCandidate GetBestLeaderCandidate(Dictionary <Guid, ElectionMessage.PrepareOk> received,
MemberInfo[] servers, Guid?resigningLeaderInstanceId, int lastAttemptedView)
{
var best = received.Values
.OrderByDescending(x => x.EpochNumber)
.ThenByDescending(x => x.LastCommitPosition)
.ThenByDescending(x => x.WriterCheckpoint)
.ThenByDescending(x => x.ChaserCheckpoint)
.ThenByDescending(x => x.NodePriority)
.ThenByDescending(x => x.ServerId)
.FirstOrDefault();
if (best == null)
{
return(null);
}
var bestCandidate = new LeaderCandidate(best.ServerId, best.ServerHttpEndPoint,
best.EpochNumber, best.EpochPosition, best.EpochId, best.EpochLeaderInstanceId,
best.LastCommitPosition, best.WriterCheckpoint, best.ChaserCheckpoint, best.NodePriority);
if (best.EpochLeaderInstanceId != Guid.Empty)
{
//best.EpochLeaderInstanceId = id of the last leader/master which has been able to write an epoch record to the transaction log and get it replicated
//to the "best" node which has the latest data, i.e data that has been replicated to at least a quorum number of nodes for sure.
//We know that the data from the "best" node originates from this leader/master itself from this epoch record onwards, thus it implies that
//the leader/master in this epoch record must have the latest replicated data as well and is thus definitely a valid candidate for leader/master if it's still alive.
//NOTE 1: It does not matter if the "best" node's latest epoch record has been replicated to a quorum number of nodes or not, the
//leader/master in the epoch record must still have the latest replicated data
//NOTE 2: it is not necessary that this leader/master is the last elected leader/master in the cluster e.g a leader/master might be elected but
//has no time to replicate the epoch. In this case, it will need to truncate when joining the new leader.
var previousLeaderId = best.EpochLeaderInstanceId;
LeaderCandidate previousLeaderCandidate = null;
//we have received a PrepareOk message from the previous leader, so we definitely know it's alive.
if (received.TryGetValue(previousLeaderId, out var leaderMsg) &&
leaderMsg.EpochNumber == best.EpochNumber &&
leaderMsg.EpochId == best.EpochId)
{
Log.Debug("ELECTIONS: (V={lastAttemptedView}) Previous Leader (L={previousLeaderId:B}) from last epoch record is still alive.", lastAttemptedView, previousLeaderId);
previousLeaderCandidate = new LeaderCandidate(leaderMsg.ServerId, leaderMsg.ServerHttpEndPoint,
leaderMsg.EpochNumber, leaderMsg.EpochPosition, leaderMsg.EpochId, leaderMsg.EpochLeaderInstanceId,
leaderMsg.LastCommitPosition, leaderMsg.WriterCheckpoint, leaderMsg.ChaserCheckpoint,
leaderMsg.NodePriority);
}
//we try to find at least one node from the quorum no. of PrepareOk messages which says that the previous leader is alive
//if none of them say it's alive, a quorum can very probably not be formed with the previous leader
//otherwise we know that the leader was alive during the last gossip time interval and we try our luck
if (previousLeaderCandidate == null)
{
foreach (var(id, prepareOk) in received)
{
var member = prepareOk.ClusterInfo.Members.FirstOrDefault(x => x.InstanceId == previousLeaderId && x.IsAlive);
if (member != null)
{
//these checks are not really necessary but we do them to ensure that everything is normal.
if (best.EpochNumber == member.EpochNumber && best.EpochId != member.EpochId)
{
//something is definitely not right if the epoch numbers match but not the epoch ids
Log.Warning("ELECTIONS: (V={lastAttemptedView}) Epoch ID mismatch in gossip information from node {nodeId:B}. Best node's Epoch Id: {bestEpochId:B}, Leader node's Epoch Id: {leaderEpochId:B}", lastAttemptedView, id, best.EpochId, member.EpochId);
continue;
}
if (best.EpochNumber - member.EpochNumber > 2)
{
//gossip information may be slightly out of date. We log a warning if the epoch number is off by more than 2
Log.Warning("ELECTIONS: (V={lastAttemptedView}) Epoch number is off by more than two in gossip information from node {nodeId:B}. Best node's Epoch number: {bestEpochNumber}, Leader node's Epoch number: {leaderEpochNumber}.", lastAttemptedView, id, best.EpochNumber, member.EpochNumber);
}
Log.Debug("ELECTIONS: (V={lastAttemptedView}) Previous Leader (L={previousLeaderId:B}) from last epoch record is still alive according to gossip from node {nodeId:B}.", lastAttemptedView, previousLeaderId, id);
previousLeaderCandidate = new LeaderCandidate(member.InstanceId,
member.HttpEndPoint,
member.EpochNumber, member.EpochPosition, member.EpochId, best.EpochLeaderInstanceId,
member.LastCommitPosition, member.WriterCheckpoint, member.ChaserCheckpoint,
member.NodePriority);
break;
}
}
}
if (previousLeaderCandidate == null)
{
Log.Debug("ELECTIONS: (V={lastAttemptedView}) Previous Leader (L={previousLeaderId:B}) from last epoch record is dead, defaulting to the best candidate (B={bestCandidateId:B}).", lastAttemptedView, previousLeaderId, bestCandidate.InstanceId);
}
else if (previousLeaderCandidate.InstanceId == resigningLeaderInstanceId)
{
Log.Debug("ELECTIONS: (V={lastAttemptedView}) Previous Leader (L={previousLeaderId:B}) from last epoch record is alive but it is resigning, defaulting to the best candidate (B={bestCandidateId:B}).", lastAttemptedView, previousLeaderId, bestCandidate.InstanceId);
}
else
{
bestCandidate = previousLeaderCandidate;
}
}
Log.Debug("ELECTIONS: (V={lastAttemptedView}) Proposing node: {leaderCandidateId:B} as best leader candidate", lastAttemptedView, bestCandidate.InstanceId);
return(bestCandidate);
}
