void LeaderTimerElapse(object userToken)
{
try
{
//Sending signal to all (except self that it is a leader)
LeaderHeartbeat heartBeat = null;
lock (lock_Operations)
{
heartBeat = new LeaderHeartbeat()
{
LeaderTerm = this.NodeTerm,
StateLogLatestIndex = NodeStateLog.StateLogId,
StateLogLatestTerm = NodeStateLog.StateLogTerm,
LastStateLogCommittedIndex = this.NodeStateLog.LastCommittedIndex,
LastStateLogCommittedIndexTerm = this.NodeStateLog.LastCommittedIndexTerm
};
}
//VerbosePrint($"{NodeAddress.NodeAddressId} (Leader)> leader_heartbeat");
this.Sender.SendToAll(eRaftSignalType.LeaderHearthbeat, heartBeat.SerializeBiser(), this.NodeAddress, entitySettings.EntityName, true);
}
catch (Exception ex)
{
Log.Log(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNode.LeaderTimerElapse"
});
}
}
public static LeaderHeartbeat BiserDecode(byte[] enc = null, Biser.Decoder extDecoder = null) //!!!!!!!!!!!!!! change return type
{
Biser.Decoder decoder = null;
if (extDecoder == null)
{
if (enc == null || enc.Length == 0)
{
return(null);
}
decoder = new Biser.Decoder(enc);
if (decoder.CheckNull())
{
return(null);
}
}
else
{
decoder = new Biser.Decoder(extDecoder);
if (decoder.IsNull)
{
return(null);
}
}
LeaderHeartbeat m = new LeaderHeartbeat(); //!!!!!!!!!!!!!! change return type
m.LeaderTerm = decoder.GetULong();
m.StateLogLatestTerm = decoder.GetULong();
m.StateLogLatestIndex = decoder.GetULong();
m.LastStateLogCommittedIndex = decoder.GetULong();
m.LastStateLogCommittedIndexTerm = decoder.GetULong();
return(m);
}
/// <summary>
/// under lock_operations
/// only for followers
/// </summary>
/// <param name="lhb"></param>
/// <returns>will return false if node needs synchronization from LastCommittedIndex/Term</returns>
public bool SetLastCommittedIndexFromLeader(LeaderHeartbeat lhb)
{
if (this.LastCommittedIndex < lhb.LastStateLogCommittedIndex)
{
//Node tries to understand if it contains already this index/term, if not it will need synchronization
ulong populateFrom = 0;
Tuple <ulong, StateLogEntry> sleTpl;
if (rn.entitySettings.DelayedPersistenceIsActive
&&
sleCache.TryGetValue(lhb.LastStateLogCommittedIndex, out sleTpl) && sleTpl.Item1 == lhb.LastStateLogCommittedIndexTerm
)
{
populateFrom = this.LastCommittedIndex + 1;
this.LastCommittedIndex = lhb.LastStateLogCommittedIndex;
this.LastCommittedIndexTerm = lhb.LastStateLogCommittedIndexTerm;
sleCacheIndex = lhb.LastStateLogCommittedIndex;
sleCacheTerm = lhb.LastStateLogCommittedIndexTerm;
}
else
{
using (var t = this.rn.db.GetTransaction())
{
t.ValuesLazyLoadingIsOn = false;
var row = t.Select <byte[], byte[]>(tblStateLogEntry, (new byte[] { 1 }).ToBytes(lhb.LastStateLogCommittedIndex, lhb.LastStateLogCommittedIndexTerm));
if (row.Exists)
{
populateFrom = this.LastCommittedIndex + 1;
this.LastCommittedIndex = lhb.LastStateLogCommittedIndex;
this.LastCommittedIndexTerm = lhb.LastStateLogCommittedIndexTerm;
//rn.VerbosePrint($"{rn.NodeAddress.NodeAddressId}> AddToLogFollower (I/T): here");
t.Insert <byte[], byte[]>(tblStateLogEntry, new byte[] { 2 }, lhb.LastStateLogCommittedIndex.ToBytes(lhb.LastStateLogCommittedIndexTerm));
t.Commit();
}
else
{
return(false);
}
}
}
if (populateFrom > 0)
{
this.rn.Commited();
}
}
return(true);
}
/// <summary>
/// under lock_operations
/// only for followers
/// </summary>
/// <param name="lhb"></param>
/// <returns>will return false if node needs synchronization from LastCommittedIndex/Term</returns>
public SyncResult SyncCommitByHeartBeat(LeaderHeartbeat lhb)
{
if (this.LastCommittedIndex < lhb.LastStateLogCommittedIndex)
{
//Node tries to understand if it contains already this index/term, if not it will need synchronization
ulong populateFrom = 0;
using (var t = this.db.GetTransaction())
{
t.ValuesLazyLoadingIsOn = false;
//find leader last commit record in database
var row = t.Select <byte[], byte[]>(stateTableName, (new byte[] { 1 }).ToBytes(lhb.LastStateLogCommittedIndex, lhb.LastStateLogCommittedIndexTerm));
if (row.Exists)
{
populateFrom = this.LastCommittedIndex + 1;
this.LastCommittedIndex = lhb.LastStateLogCommittedIndex;
this.LastCommittedIndexTerm = lhb.LastStateLogCommittedIndexTerm;
t.Insert <byte[], byte[]>(stateTableName, new byte[] { 2 }, lhb.LastStateLogCommittedIndex.ToBytes(lhb.LastStateLogCommittedIndexTerm));
t.Commit();
}
else
{
return new SyncResult()
{
Synced = false,
HasCommit = false,
}
};
}
if (populateFrom > 0)
{
return new SyncResult()
{
Synced = true,
HasCommit = true,
}
}
;
}
return(new SyncResult()
{
Synced = true,
HasCommit = false,
});
}
/// <summary>
/// Leader receives accepted Log
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntryAccepted(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Leader)
{
return;
}
StateLogEntryApplied applied = StateLogEntryApplied.BiserDecode(data);
var res = this.NodeStateLog.EntryIsAccepted(address, GetMajorityQuantity(), applied);
if (res == StateLog.eEntryAcceptanceResult.Committed)
{
this.VerbosePrint($"{this.NodeAddress.NodeAddressId}> LogEntry {applied.StateLogEntryId} is COMMITTED (answer from {address.NodeAddressId})");
RemoveLeaderLogResendTimer();
//Force heartbeat, to make followers to get faster info about commited elements
LeaderHeartbeat heartBeat = new LeaderHeartbeat()
{
LeaderTerm = this.NodeTerm,
StateLogLatestIndex = NodeStateLog.StateLogId,
StateLogLatestTerm = NodeStateLog.StateLogTerm,
LastStateLogCommittedIndex = this.NodeStateLog.LastCommittedIndex,
LastStateLogCommittedIndexTerm = this.NodeStateLog.LastCommittedIndexTerm
};
this.Sender.SendToAll(eRaftSignalType.LeaderHearthbeat, heartBeat.SerializeBiser(), this.NodeAddress, entitySettings.EntityName, true);
//---------------------------------------
//this.NodeStateLog.RemoveEntryFromDistribution(applied.StateLogEntryId, applied.StateLogEntryTerm);
InLogEntrySend = false;
ApplyLogEntry();
}
}
/// <summary>
/// under lock_operations
/// only for followers
/// </summary>
/// <param name="lhb"></param>
/// <returns>will return false if node needs synchronization from LastCommittedIndex/Term</returns>
public bool SetLastCommittedIndexFromLeader(LeaderHeartbeat lhb)
{
if (this.LastCommittedIndex < lhb.LastStateLogCommittedIndex)
{
//Node tries to understand if it contains already this index/term, if not it will need synchronization
ulong populateFrom = 0;
//find leader last commit record in database
var col = this.db.GetCollection <StateLogEntry>(stateTableName);
var row = col
.Query()
.Where(x => x.Index == lhb.LastStateLogCommittedIndex && x.Term == lhb.LastStateLogCommittedIndexTerm)
.FirstOrDefault();
//var row = t.Select<byte[], byte[]>(stateTableName, (new byte[] { 1 }).ToBytes(lhb.LastStateLogCommittedIndex, lhb.LastStateLogCommittedIndexTerm));
if (row != null && row.IsCommitted == false)
{
populateFrom = this.LastCommittedIndex + 1;
this.LastCommittedIndex = lhb.LastStateLogCommittedIndex;
this.LastCommittedIndexTerm = lhb.LastStateLogCommittedIndexTerm;
row.IsCommitted = true;
col.Update(row);
//t.Insert<byte[], byte[]>(stateTableName, new byte[] { 2 }, lhb.LastStateLogCommittedIndex.ToBytes(lhb.LastStateLogCommittedIndexTerm));
//t.Commit();
}
else
{
return(false);
}
if (populateFrom > 0)
{
this.statemachine.logHandler.Commited();
}
}
return(true);
}
/// <summary>
/// Is called from lock_Operations and try catch
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseLeaderHeartbeat(NodeAddress address, byte[] data)
{
//var LeaderHeartbeat = data.DeserializeProtobuf<LeaderHeartbeat>();
this.LeaderHeartbeat = LeaderHeartbeat.BiserDecode(data); //data.DeserializeProtobuf<LeaderHeartbeat>();
// Setting variable of the last heartbeat
this.LeaderHeartbeatArrivalTime = DateTime.Now;
this.LeaderNodeAddress = address; //Can be incorrect in case if this node is Leader, must
//Comparing Terms
if (this.NodeTerm < LeaderHeartbeat.LeaderTerm)
{
this.NodeTerm = LeaderHeartbeat.LeaderTerm;
switch (this.NodeState)
{
case eNodeState.Leader:
//Stepping back from Leader to Follower
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Follower:
//Ignoring
SetNodeFollower(); //Reseting timers
break;
}
}
else
{
switch (this.NodeState)
{
case eNodeState.Leader:
//2 leaders with the same Term
if (this.NodeTerm > LeaderHeartbeat.LeaderTerm)
{
//Ignoring
//Incoming signal is not from the Leader anymore
return;
}
else
{
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
}
break;
case eNodeState.Candidate:
//Stepping back
SetNodeFollower();
VerbosePrint("Node {0} state is {1} _IncomingSignalHandler", NodeAddress.NodeAddressId, this.NodeState);
break;
case eNodeState.Follower:
SetNodeFollower();
break;
}
}
//Here will come only Followers
this.LeaderNodeAddress = address;
if (!IsLeaderSynchroTimerActive && !this.NodeStateLog.SetLastCommittedIndexFromLeader(LeaderHeartbeat))
{
//VerbosePrint($"{NodeAddress.NodeAddressId}> in sync 2 ");
this.SyncronizeWithLeader();
}
}