/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public bool CommitLogEntry(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (this.LastCommittedIndex < applied.StateLogEntryId && statemachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
t.Insert <byte[], byte[]>(stateTableName, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
//qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
return(lstCommited.Count > 0);
}
return(false);
}
/// <summary>
/// Node receives answer votes (to become Leader) from other nodes.
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="data"></param>
void ParseVoteOfCandidate(NodeRaftAddress address, object data)
{
//Node received a node
var vote = data as VoteOfCandidate;
var termState = CompareCurrentTermWithIncoming(vote.TermId);
if (this.States.NodeState != eNodeState.Candidate)
{
return;
}
switch (vote.VoteType)
{
case VoteOfCandidate.eVoteType.VoteFor:
//Calculating if node has Majority of
//VotesQuantity++;
States.VotesQuantity.Add(address.EndPointSID);
if ((States.VotesQuantity.Count + 1) >= this.GetMajorityQuantity())
{
//Majority
//Node becomes a Leader
this.States.NodeState = eNodeState.Leader;
//this.NodeStateLog.FlushSleCache();
this.logHandler.ClearLogAcceptance();
this.logHandler.ClearLogEntryForDistribution();
VerbosePrint("Node {0} state is {1} _ParseVoteOfCandidate", NodeAddress.NodeAddressId, this.States.NodeState);
VerbosePrint("Node {0} is Leader **********************************************", NodeAddress.NodeAddressId);
//Stopping timers
this.timerLoop.StopElectionTimeLoop();
this.timerLoop.StopLeaderHeartbeatWaitingTimeLoop();
/*
* It's possible that we receive higher term from another leader
* (in case if this leader was disconnected for some seconds from the network,
* other leader can be elected and it will definitely have higher Term, so every Leader node must be ready to it)
*/
this.timerLoop.EnterLeaderLoop();
}
//else
//{
// //Accumulating voices
// //Do nothing
//}
break;
case VoteOfCandidate.eVoteType.VoteReject:
//Do nothing
break;
}
}
/// <summary>
///
/// </summary>
/// <param name="address">Address of the node who sent the signal</param>
/// <param name="signalType"></param>
/// <param name="data"></param>
public void HandleRaftSignal(NodeRaftAddress address, eRaftSignalType signalType, object data)
{
try
{
if (GlobalConfig.Verbose)
{
Console.WriteLine("mesage from:" + address.EndPointSID + " signal:" + signalType);
}
lock (lock_Operations)
{
switch (signalType)
{
case eRaftSignalType.LeaderHearthbeat:
ParseLeaderHeartbeat(address, data);
break;
case eRaftSignalType.CandidateRequest:
ParseCandidateRequest(address, data);
break;
case eRaftSignalType.VoteOfCandidate:
ParseVoteOfCandidate(address, data);
break;
case eRaftSignalType.StateLogEntrySuggestion:
ParseStateLogEntrySuggestion(address, data);
break;
case eRaftSignalType.StateLogEntryRequest:
ParseStateLogEntryRequest(address, data);
break;
case eRaftSignalType.StateLogEntryAccepted:
this.logHandler.ParseStateLogEntryAccepted(address, data);
break;
case eRaftSignalType.StateLogRedirectRequest: //Not a leader node tries to add command
this.logHandler.ParseStateLogRedirectRequest(address, data);
break;
}
}
}
catch (Exception ex)
{
Log.Log(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNode.IncomingSignalHandler"
});
}
}
/// <summary>
/// Only for Follower
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntrySuggestion(NodeRaftAddress address, object data)
{
if (this.States.NodeState != eNodeState.Follower)
{
return;
}
StateLogEntrySuggestion suggest = data as StateLogEntrySuggestion;
if (this.NodeTerm > suggest.LeaderTerm) //Sending Leader is not Leader anymore
{
this.States.LeaderSynchronizationIsActive = false;
return;
}
if (this.NodeTerm < suggest.LeaderTerm)
{
this.NodeTerm = suggest.LeaderTerm;
}
if (suggest.StateLogEntry == null || (suggest.StateLogEntry.Index <= NodeStateLog.LastCommittedIndex)) //Preventing same entry income, can happen if restoration was sent twice (while switch of leaders)
{
return; //breakpoint don't remove
}
//Checking if node can accept current suggestion
if (suggest.StateLogEntry.PreviousStateLogId > 0)
{
var sle = this.NodeStateLog.GetEntryByIndexTerm(suggest.StateLogEntry.PreviousStateLogId, suggest.StateLogEntry.PreviousStateLogTerm);
if (sle == null)
{
this.SyncronizeWithLeader();
return;
}
}
//We can apply new Log Entry from the Leader and answer successfully
this.logHandler.AddLogEntryByFollower(suggest);
StateLogEntryApplied applied = new StateLogEntryApplied()
{
StateLogEntryId = suggest.StateLogEntry.Index,
StateLogEntryTerm = suggest.StateLogEntry.Term
// RedirectId = suggest.StateLogEntry.RedirectId
};
this.network.SendTo(address, eRaftSignalType.StateLogEntryAccepted, applied, this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// Only for Leader.
/// Follower requests new Log Entry Index from the Leader and Leader answers to the Follower
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntryRequest(NodeRaftAddress address, object data)
{
if (this.States.NodeState != eNodeState.Leader)
{
return;
}
StateLogEntryRequest req = data as StateLogEntryRequest;
//Getting suggestion
var suggestion = this.NodeStateLog.GetNextStateLogEntrySuggestion(req);
if (GlobalConfig.Verbose)
{
Console.WriteLine($"create suggestion to req:{req.StateLogEntryId}" + suggestion.StateLogEntry.Index + " is commit:" + suggestion.StateLogEntry.IsCommitted);
}
//VerbosePrint($"{NodeAddress.NodeAddressId} (Leader)> Request (I): {req.StateLogEntryId} from {address.NodeAddressId};");
if (suggestion != null)
{
this.network.SendTo(address, eRaftSignalType.StateLogEntrySuggestion, suggestion, this.NodeAddress, entitySettings.EntityName);
}
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public eEntryAcceptanceResult EntryIsAccepted(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (applied.StateLogEntryId <= this.LastCommittedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
if (applied.StateLogEntryId <= this.LastAppliedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
StateLogEntryAcceptance acc = null;
if (dStateLogEntryAcceptance.TryGetValue(applied.StateLogEntryId, out acc))
{
if (acc.Term != applied.StateLogEntryTerm)
{
return(eEntryAcceptanceResult.NotAccepted); //Came from wrong Leader probably
}
acc.acceptedEndPoints.Add(address.EndPointSID);
}
else
{
acc = new StateLogEntryAcceptance()
{
Index = applied.StateLogEntryId,
Term = applied.StateLogEntryTerm
};
acc.acceptedEndPoints.Add(address.EndPointSID);
dStateLogEntryAcceptance[applied.StateLogEntryId] = acc;
}
if ((acc.acceptedEndPoints.Count + 1) >= majorityQuantity)
{
this.LastAppliedIndex = applied.StateLogEntryId;
//Removing from Dictionary
dStateLogEntryAcceptance.Remove(applied.StateLogEntryId);
if (this.LastCommittedIndex < applied.StateLogEntryId && statemachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
var col = this.db.GetCollection <StateLogEntry>(stateTableName);
var list = col.Query().Where(s => s.Index >= this.LastCommittedIndex + 1 && s.Term == applied.StateLogEntryTerm).ToList();
foreach (var item in list)
{
lstCommited.Add(new byte[1]);
item.IsCommitted = true;
col.Update(item);
qDistribution.Remove(applied.StateLogEntryId);
}
//using (var t = this.db.GetTransaction())
//{
// //Gathering all not commited entries that are bigger than latest committed index
// t.ValuesLazyLoadingIsOn = false;
// foreach (var el in t.SelectForwardFromTo<byte[], byte[]>(stateTableName,
// new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
// new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
// {
// lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
// }
// t.Insert<byte[], byte[]>(stateTableName, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
// t.Commit();
// qDistribution.Remove(applied.StateLogEntryId);
//}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
if (lstCommited.Count > 0)
{
this.statemachine.logHandler.Commited();
}
return(eEntryAcceptanceResult.Committed);
}
}
return(eEntryAcceptanceResult.Accepted);
}
/// <summary>
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="data"></param>
void ParseCandidateRequest(NodeRaftAddress address, object data)
{
var req = data as CandidateRequest;
VoteOfCandidate vote = new VoteOfCandidate();
vote.VoteType = VoteOfCandidate.eVoteType.VoteFor;
var termState = CompareCurrentTermWithIncoming(req.TermId);
vote.TermId = NodeTerm;
switch (termState)
{
case eTermComparationResult.CurrentTermIsHigher:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eTermComparationResult.CurrentTermIsSmaller:
//Now this Node is Follower
break;
}
if (vote.VoteType == VoteOfCandidate.eVoteType.VoteFor)
{
switch (this.States.NodeState)
{
case eNodeState.Leader:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eNodeState.Candidate:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eNodeState.Follower:
//Probably we can vote for this Node (if we didn't vote for any other one)
if (LastVotedTermId < req.TermId)
{
//formula of voting
if (
(NodeStateLog.StateLogTerm > req.LastTermId)
||
(
NodeStateLog.StateLogTerm == req.LastTermId
&&
NodeStateLog.StateLogId > req.LastLogId
)
)
{
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
}
else
{
LastVotedTermId = req.TermId;
vote.VoteType = VoteOfCandidate.eVoteType.VoteFor;
//Restaring Election Timer
this.timerLoop.StopElectionTimeLoop();
this.timerLoop.EnterElectionTimeLoop();
}
}
else
{
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
}
break;
}
}
//Sending vote signal back
//VerbosePrint("Node {0} voted to node {1} as {2} _ParseCandidateRequest", NodeAddress.NodeAddressId, address.NodeAddressId, vote.VoteType);
VerbosePrint($"Node {NodeAddress.NodeAddressId} ({this.States.NodeState}) {vote.VoteType} {address.NodeAddressId} in _ParseCandidateRequest");
network.SendTo(address, eRaftSignalType.VoteOfCandidate, vote, this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// Is called from lock_Operations and try catch
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseLeaderHeartbeat(NodeRaftAddress address, object data)
{
//var LeaderHeartbeat = data.DeserializeProtobuf<LeaderHeartbeat>();
this.States.LeaderHeartbeat = data as LeaderHeartbeat; //data.DeserializeProtobuf<LeaderHeartbeat>();
// Setting variable of the last heartbeat
this.States.LeaderHeartbeatArrivalTime = DateTime.Now;
this.LeaderNodeAddress = address; //Can be incorrect in case if this node is Leader, must
//Comparing Terms
if (this.NodeTerm < States.LeaderHeartbeat.LeaderTerm)
{
this.NodeTerm = States.LeaderHeartbeat.LeaderTerm;
switch (this.States.NodeState)
{
case eNodeState.Leader:
//Stepping back from Leader to Follower
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.States.NodeState);
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.States.NodeState);
break;
case eNodeState.Follower:
//Ignoring
SetNodeFollower(); //Reseting timers
break;
}
}
else
{
switch (this.States.NodeState)
{
case eNodeState.Leader:
//2 leaders with the same Term
if (this.NodeTerm > States.LeaderHeartbeat.LeaderTerm)
{
//Ignoring
//Incoming signal is not from the Leader anymore
return;
}
else
{
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.States.NodeState);
}
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.States.NodeState);
break;
case eNodeState.Follower:
SetNodeFollower();
break;
}
}
//Here will come only Followers
this.LeaderNodeAddress = address;
var result = this.NodeStateLog.SyncCommitByHeartBeat(this.States.LeaderHeartbeat);
if (result.HasCommit)
{
this.logHandler.Commited();
}
if (!IsLeaderSynchroTimerActive && !result.Synced)
{
VerbosePrint($"{NodeAddress.NodeAddressId} start sync ");
this.SyncronizeWithLeader();
}
}