/// <summary>
/// Only for Follower
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntrySuggestion(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Follower)
{
return;
}
StateLogEntrySuggestion suggest = StateLogEntrySuggestion.BiserDecode(data); //data.DeserializeProtobuf<StateLogEntrySuggestion>();
if (this.NodeTerm > suggest.LeaderTerm) //Sending Leader is not Leader anymore
{
this.NodeStateLog.LeaderSynchronizationIsActive = false;
return;
}
if (this.NodeTerm < suggest.LeaderTerm)
{
this.NodeTerm = suggest.LeaderTerm;
}
if (suggest.StateLogEntry.Index <= NodeStateLog.LastCommittedIndex) //Preventing same entry income, can happen if restoration was sent twice (while switch of leaders)
{
return; //breakpoint don't remove
}
//Checking if node can accept current suggestion
if (suggest.StateLogEntry.PreviousStateLogId > 0)
{
var sle = this.NodeStateLog.GetEntryByIndexTerm(suggest.StateLogEntry.PreviousStateLogId, suggest.StateLogEntry.PreviousStateLogTerm);
if (sle == null)
{
//We don't have previous to this log and need new index request
//VerbosePrint($"{NodeAddress.NodeAddressId}> in sync 1 ");
if (entitySettings.InMemoryEntity && entitySettings.InMemoryEntityStartSyncFromLatestEntity && this.NodeStateLog.LastAppliedIndex == 0)
{
//helps newly starting mode with specific InMemory parameters get only latest command for the entity
}
else
{
this.SyncronizeWithLeader();
return;
}
}
}
//We can apply new Log Entry from the Leader and answer successfully
this.NodeStateLog.AddToLogFollower(suggest);
StateLogEntryApplied applied = new StateLogEntryApplied()
{
StateLogEntryId = suggest.StateLogEntry.Index,
StateLogEntryTerm = suggest.StateLogEntry.Term
// RedirectId = suggest.StateLogEntry.RedirectId
};
//this.NodeStateLog.LeaderSynchronizationIsActive = false;
this.Sender.SendTo(address, eRaftSignalType.StateLogEntryAccepted, applied.SerializeBiser(), this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// Node receives answer votes (to become Leader) from other nodes.
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="data"></param>
void ParseVoteOfCandidate(NodeAddress address, byte[] data)
{
//Node received a node
var vote = VoteOfCandidate.BiserDecode(data);
var termState = CompareCurrentTermWithIncoming(vote.TermId);
if (this.NodeState != eNodeState.Candidate)
{
return;
}
switch (vote.VoteType)
{
case VoteOfCandidate.eVoteType.VoteFor:
//Calculating if node has Majority of
//VotesQuantity++;
VotesQuantity.Add(address.EndPointSID);
if ((VotesQuantity.Count + 1) >= this.GetMajorityQuantity())
{
//Majority
//Node becomes a Leader
this.NodeState = eNodeState.Leader;
this.NodeStateLog.FlushSleCache();
this.NodeStateLog.ClearLogAcceptance();
this.NodeStateLog.ClearLogEntryForDistribution();
VerbosePrint("Node {0} state is {1} _ParseVoteOfCandidate", NodeAddress.NodeAddressId, this.NodeState);
VerbosePrint("Node {0} is Leader **********************************************", NodeAddress.NodeAddressId);
//Stopping timers
this.RemoveElectionTimer();
this.RemoveLeaderHeartbeatWaitingTimer();
/*
* It's possible that we receive higher term from another leader
* (in case if this leader was disconnected for some seconds from the network,
* other leader can be elected and it will definitely have higher Term, so every Leader node must be ready to it)
*/
this.RunLeaderTimer();
}
//else
//{
// //Accumulating voices
// //Do nothing
//}
break;
case VoteOfCandidate.eVoteType.VoteReject:
//Do nothing
break;
}
}
/// <summary>
/// Is called from RaftNode lock_Operations
/// </summary>
public ulong StoreRedirect(NodeAddress na)
{
redirectId++;
_d[redirectId] = new RedirectInfo()
{
Term = rn.NodeTerm,
NodeAddress = na
};
//...store here callbacks in dictionary also term that shoud be checked before returning redirect id
//make auto clean by timeout etc...
return(redirectId);
}
/// <summary>
///
/// </summary>
/// <param name="nodeAddress"></param>
/// <param name="signalType"></param>
/// <param name="data"></param>
/// <param name="senderNodeAddress"></param>
public void SendTo(NodeAddress nodeAddress, eRaftSignalType signalType, byte[] data, NodeAddress senderNodeAddress)
{
try
{
//!!!!!!!!!!!!!!!!!!!! GO ON HERE
}
catch (Exception ex)
{
this.LogError(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNodeUdp.SendTo"
});
}
}
/// <summary>
/// called from lock try..catch
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogRedirectRequest(NodeAddress address, byte[] data)
{
StateLogEntryRedirectRequest req = StateLogEntryRedirectRequest.BiserDecode(data);
//StateLogEntryRedirectResponse resp = new StateLogEntryRedirectResponse(); //{ RedirectId = req.RedirectId };
if (this.NodeState != eNodeState.Leader) //Just return
{
return;
}
this.NodeStateLog.AddStateLogEntryForDistribution(req.Data);//, redirectId);
ApplyLogEntry();
//Don't answer, committed value wil be delivered via standard channel
}
/// <summary>
///
/// </summary>
/// <param name="address">Address of the node who sent the signal</param>
/// <param name="signalType"></param>
/// <param name="data"></param>
public void IncomingSignalHandler(NodeAddress address, eRaftSignalType signalType, byte[] data)
{
try
{
lock (lock_Operations)
{
switch (signalType)
{
case eRaftSignalType.LeaderHearthbeat:
ParseLeaderHeartbeat(address, data);
break;
case eRaftSignalType.CandidateRequest:
ParseCandidateRequest(address, data);
break;
case eRaftSignalType.VoteOfCandidate:
ParseVoteOfCandidate(address, data);
break;
case eRaftSignalType.StateLogEntrySuggestion:
ParseStateLogEntrySuggestion(address, data);
break;
case eRaftSignalType.StateLogEntryRequest:
ParseStateLogEntryRequest(address, data);
break;
case eRaftSignalType.StateLogEntryAccepted:
ParseStateLogEntryAccepted(address, data);
break;
case eRaftSignalType.StateLogRedirectRequest: //Not a leader node tries to add command
ParseStateLogRedirectRequest(address, data);
break;
}
}
}
catch (Exception ex)
{
Log.Log(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNode.IncomingSignalHandler"
});
}
}
/// <summary>
/// Only for Leader.
/// Follower requests new Log Entry Index from the Leader and Leader answers to the Follower
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntryRequest(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Leader)
{
return;
}
StateLogEntryRequest req = StateLogEntryRequest.BiserDecode(data);//.DeserializeProtobuf<StateLogEntryRequest>();
//Getting suggestion
var suggestion = this.NodeStateLog.GetNextStateLogEntrySuggestionFromRequested(req);
//VerbosePrint($"{NodeAddress.NodeAddressId} (Leader)> Request (I/T): {req.StateLogEntryId}/{req.StateLogEntryTerm} from {address.NodeAddressId};");
VerbosePrint($"{NodeAddress.NodeAddressId} (Leader)> Request (I): {req.StateLogEntryId} from {address.NodeAddressId};");
if (suggestion != null)
{
this.Sender.SendTo(address, eRaftSignalType.StateLogEntrySuggestion, suggestion.SerializeBiser(), this.NodeAddress, entitySettings.EntityName);
}
}
/// <summary>
///
/// </summary>
/// <param name="signalType"></param>
/// <param name="data"></param>
/// <param name="senderNodeAddress"></param>
public void SendToAll(eRaftSignalType signalType, byte[] data, NodeAddress senderNodeAddress)
{
try
{
List <NodeAddress> lst = null;
sync_ListClusterEndPoints.EnterReadLock();
try
{
lst = ListClusterEndPoints.Where(r => !r.Value.IsMe).Select(r => r.Value).ToList();
}
finally
{
sync_ListClusterEndPoints.ExitReadLock();
}
if (lst == null || lst.Count() < 1)
{
return;
}
foreach (var n in lst)
{
//!!!!!!!!!!!!!!!!!!!! GO ON HERE
//udpSocket.SendTo(n.IpEP,signalType,data)
//////////if (n.NodeAddress.NodeAddressId == nodeAddress.NodeAddressId)
//////////{
////////// //May be put it all into new Threads or so
////////// ((IRaftComReceiver)n).IncomingSignalHandler(myNodeAddress, signalType, data);
////////// break;
//////////}
}
}
catch (Exception ex)
{
this.LogError(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNodeUdp.SendToAll"
});
}
}
/// <summary>
/// Leader receives accepted Log
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntryAccepted(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Leader)
{
return;
}
StateLogEntryApplied applied = StateLogEntryApplied.BiserDecode(data);
var res = this.NodeStateLog.EntryIsAccepted(address, GetMajorityQuantity(), applied);
if (res == StateLog.eEntryAcceptanceResult.Committed)
{
this.VerbosePrint($"{this.NodeAddress.NodeAddressId}> LogEntry {applied.StateLogEntryId} is COMMITTED (answer from {address.NodeAddressId})");
RemoveLeaderLogResendTimer();
//Force heartbeat, to make followers to get faster info about commited elements
LeaderHeartbeat heartBeat = new LeaderHeartbeat()
{
LeaderTerm = this.NodeTerm,
StateLogLatestIndex = NodeStateLog.StateLogId,
StateLogLatestTerm = NodeStateLog.StateLogTerm,
LastStateLogCommittedIndex = this.NodeStateLog.LastCommittedIndex,
LastStateLogCommittedIndexTerm = this.NodeStateLog.LastCommittedIndexTerm
};
this.Sender.SendToAll(eRaftSignalType.LeaderHearthbeat, heartBeat.SerializeBiser(), this.NodeAddress, entitySettings.EntityName, true);
//---------------------------------------
//this.NodeStateLog.RemoveEntryFromDistribution(applied.StateLogEntryId, applied.StateLogEntryTerm);
InLogEntrySend = false;
ApplyLogEntry();
}
}
/// <summary>
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="data"></param>
void ParseCandidateRequest(NodeAddress address, byte[] data)
{
var req = CandidateRequest.BiserDecode(data);
VoteOfCandidate vote = new VoteOfCandidate();
vote.VoteType = VoteOfCandidate.eVoteType.VoteFor;
var termState = CompareCurrentTermWithIncoming(req.TermId);
vote.TermId = NodeTerm;
switch (termState)
{
case eTermComparationResult.CurrentTermIsHigher:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eTermComparationResult.CurrentTermIsSmaller:
//Now this Node is Follower
break;
}
if (vote.VoteType == VoteOfCandidate.eVoteType.VoteFor)
{
switch (this.NodeState)
{
case eNodeState.Leader:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eNodeState.Candidate:
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
break;
case eNodeState.Follower:
//Probably we can vote for this Node (if we didn't vote for any other one)
if (LastVotedTermId < req.TermId)
{
//formula of voting
if (
(NodeStateLog.StateLogTerm > req.LastTermId)
||
(
NodeStateLog.StateLogTerm == req.LastTermId
&&
NodeStateLog.StateLogId > req.LastLogId
)
)
{
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
}
else
{
LastVotedTermId = req.TermId;
vote.VoteType = VoteOfCandidate.eVoteType.VoteFor;
//Restaring Election Timer
this.RemoveElectionTimer();
this.RunElectionTimer();
}
}
else
{
vote.VoteType = VoteOfCandidate.eVoteType.VoteReject;
}
break;
}
}
//Sending vote signal back
//VerbosePrint("Node {0} voted to node {1} as {2} _ParseCandidateRequest", NodeAddress.NodeAddressId, address.NodeAddressId, vote.VoteType);
VerbosePrint($"Node {NodeAddress.NodeAddressId} ({this.NodeState}) {vote.VoteType} {address.NodeAddressId} in _ParseCandidateRequest");
Sender.SendTo(address, eRaftSignalType.VoteOfCandidate, vote.SerializeBiser(), this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// Is called from lock_Operations and try catch
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseLeaderHeartbeat(NodeAddress address, byte[] data)
{
//var LeaderHeartbeat = data.DeserializeProtobuf<LeaderHeartbeat>();
this.LeaderHeartbeat = LeaderHeartbeat.BiserDecode(data); //data.DeserializeProtobuf<LeaderHeartbeat>();
// Setting variable of the last heartbeat
this.LeaderHeartbeatArrivalTime = DateTime.Now;
this.LeaderNodeAddress = address; //Can be incorrect in case if this node is Leader, must
//Comparing Terms
if (this.NodeTerm < LeaderHeartbeat.LeaderTerm)
{
this.NodeTerm = LeaderHeartbeat.LeaderTerm;
switch (this.NodeState)
{
case eNodeState.Leader:
//Stepping back from Leader to Follower
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Follower:
//Ignoring
SetNodeFollower(); //Reseting timers
break;
}
}
else
{
switch (this.NodeState)
{
case eNodeState.Leader:
//2 leaders with the same Term
if (this.NodeTerm > LeaderHeartbeat.LeaderTerm)
{
//Ignoring
//Incoming signal is not from the Leader anymore
return;
}
else
{
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
}
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Follower:
SetNodeFollower();
break;
}
}
//Here will come only Followers
this.LeaderNodeAddress = address;
if (!IsLeaderSynchroTimerActive && !this.NodeStateLog.SetLastCommittedIndexFromLeader(LeaderHeartbeat))
{
//VerbosePrint($"{NodeAddress.NodeAddressId}> in sync 2 ");
this.SyncronizeWithLeader();
}
}
/// <summary>
/// Time to become a candidate
/// </summary>
/// <param name="userToken"></param>
void ElectionTimeout(object userToken)
{
CandidateRequest req = null;
try
{
lock (lock_Operations)
{
if (Election_TimerId == 0) //Timer was switched off and we don't need to run it again
{
return;
}
Election_TimerId = 0;
if (this.NodeState == eNodeState.Leader)
{
return;
}
VerbosePrint("Node {0} election timeout", NodeAddress.NodeAddressId);
this.NodeState = eNodeState.Candidate;
this.LeaderNodeAddress = null;
VerbosePrint("Node {0} state is {1} _ElectionTimeout", NodeAddress.NodeAddressId, this.NodeState);
//Voting for self
//VotesQuantity = 1;
VotesQuantity.Clear();
//Increasing local term number
NodeTerm++;
req = new CandidateRequest()
{
TermId = this.NodeTerm,
LastLogId = NodeStateLog.StateLogId,
LastTermId = NodeStateLog.StateLogTerm
};
//send to all was here
//Setting up new Election Timer
RunElectionTimer();
}
this.Sender.SendToAll(eRaftSignalType.CandidateRequest, req.SerializeBiser(), this.NodeAddress, entitySettings.EntityName);
}
catch (Exception ex)
{
Log.Log(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNode.ElectionTimeout"
});
}
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public eEntryAcceptanceResult EntryIsAccepted(NodeAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (applied.StateLogEntryId <= this.LastCommittedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
if (applied.StateLogEntryId <= this.LastAppliedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
StateLogEntryAcceptance acc = null;
if (dStateLogEntryAcceptance.TryGetValue(applied.StateLogEntryId, out acc))
{
if (acc.Term != applied.StateLogEntryTerm)
{
return(eEntryAcceptanceResult.NotAccepted); //Came from wrong Leader probably
}
//acc.Quantity += 1;
acc.acceptedEndPoints.Add(address.EndPointSID);
}
else
{
acc = new StateLogEntryAcceptance()
{
//Quantity = 2, //Leader + first incoming
Index = applied.StateLogEntryId,
Term = applied.StateLogEntryTerm
};
acc.acceptedEndPoints.Add(address.EndPointSID);
dStateLogEntryAcceptance[applied.StateLogEntryId] = acc;
}
if ((acc.acceptedEndPoints.Count + 1) >= majorityQuantity)
{
this.LastAppliedIndex = applied.StateLogEntryId;
//Removing from Dictionary
dStateLogEntryAcceptance.Remove(applied.StateLogEntryId);
if (this.LastCommittedIndex < applied.StateLogEntryId && rn.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.rn.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
//Setting latest commited index
if (rn.entitySettings.DelayedPersistenceIsActive)
{
foreach (var iel in sleCache.Where(r => r.Key >= this.LastCommittedIndex + 1 && r.Value.Item1 == applied.StateLogEntryTerm))
{
lstCommited.Add(iel.Value.Item2.Data);
}
sleCacheIndex = applied.StateLogEntryId;
sleCacheTerm = applied.StateLogEntryTerm;
}
else
{
t.Insert <byte[], byte[]>(tblStateLogEntry, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
}
//Removing entry from command queue
//t.RemoveKey<byte[]>(tblAppendLogEntry, new byte[] { 1 }.ToBytes(applied.StateLogEntryTerm, applied.StateLogEntryId));
qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
if (lstCommited.Count > 0)
{
this.rn.Commited();
}
//this.rn.Commited(applied.StateLogEntryId);
return(eEntryAcceptanceResult.Committed);
}
}
return(eEntryAcceptanceResult.Accepted);
}