public void RaciveHartbeat(AppendEntryMessage appendEntry)
{
if (_serverIdentifier.Equals(appendEntry.Leader))
return;
if (_options.UseLogging)
_logger.LogInformation($"Processing {nameof(RaciveHartbeat)} \n\t\t Leader {appendEntry.Leader} \n\t\t Term {appendEntry.Term}");
// If not in your term reject request
if (appendEntry.Term < _election.CurrentTerm)
{
_entryReply.OnNext(new AppendEntryResultMessage()
{
Term = _election.CurrentTerm,
Success = false,
});
return;
}
// Update to follow new term
if (appendEntry.Term > _election.CurrentTerm)
{
_election.CurrentTerm = appendEntry.Term;
_election.VotedFor = appendEntry.Leader;
_state?.OnNext(ServerStateType.Follower);
_entryReply.OnNext(new AppendEntryResultMessage()
{
Term = _election.CurrentTerm,
Success = false,
});
return;
}
_hartbeat.Reset();
var term = _election.CurrentTerm;
var outOfSync = false;
if (_logReplication.Contains(appendEntry.PreviousLogIndex))
{
// If the appendEntry from the leader is in sync with us, the follower
if (_logReplication.Term(appendEntry.PreviousLogIndex) == appendEntry.PreviousLogTerm)
{
// If we need to rollback the logs
var lastApplied = appendEntry.PreviousLogIndex + 1;
if (lastApplied != _logReplication.LastApplied)
{
_logReplication.Rollback(lastApplied);
}
// We can start processing the logs
var successes = new List<bool>();
if (appendEntry.Entries != null)
{
foreach (var log in appendEntry.Entries)
{
successes.Add(_logReplication.Append(log));
}
}
// Confirm Log append
_entryReply.OnNext(new AppendEntryResultMessage()
{
Term = term,
LogIndex = _logReplication.LastApplied,
LogTerm = _logReplication.LastTerm,
Success = successes.TrueForAll(p => p),
From = _serverIdentifier
});
}
else
{
outOfSync = true;
}
}
else
{
outOfSync = true;
}
// reject the append entry as the leader is out of sync with the follower
if (outOfSync)
{
// Will keep moving the log index back until the followr and the leader match on the log consistency
// This will let the lead overwrite the followers logs to bring it back into full consistency.
var previousIndex = _logReplication.LastApplied;
if (appendEntry.PreviousLogIndex <= previousIndex)
{
previousIndex = appendEntry.PreviousLogIndex - 1;
}
var previousTerm = _logReplication.Term(previousIndex);
// update the lead with our last index and term of a successfully append
_entryReply.OnNext(new AppendEntryResultMessage()
{
Term = term,
LogIndex = previousIndex,
LogTerm = previousTerm,
Success = false,
From = _serverIdentifier
});
}
var logsToProcess = _logReplication.ToCommit(appendEntry.LeaderCommit, appendEntry.Term);
// Process the logs to commit as a task so not to block RPC
ProcessLogs(logsToProcess);
}
public void SendHartbeat(ServerStateType type)
{
if (_isDispose) return;
var term = _election.CurrentTerm;
var votedFor = _election.VotedFor;
foreach (var node in _nodes)
{
if (_isDispose) return;
if (!NextIndex.ContainsKey(node))
{
// Adds a node to our record and assumes it's log state is the same as our's
// If it's not the same the follower will reject the message and send
// back it's current state for the next heartbeat
NextIndex.TryAdd(node, _logReplication.LastApplied);
MatchIndex.TryAdd(node, 0);
}
var next = NextIndex[node];
var previousLogIndex = next - 1;
var entries = _logReplication.Replicate(next);
var logEntries = entries as IList<LogEntry> ?? entries.ToList();
var message = new AppendEntryMessage()
{
To = node,
Term = term,
Leader = votedFor,
Entries = logEntries,
LeaderCommit = _logReplication.CommitIndex,
PreviousLogIndex = previousLogIndex,
PreviousLogTerm = _logReplication.Term(previousLogIndex)
};
if (_options.UseLogging)
_logger.LogInformation($"Sending {nameof(AppendEntryMessage)} \n\t\t Term {message.Term} \n\t\t To {message.To} \n\t\t Leader {message.Leader} \n\t\t Entries {logEntries.Count()}");
_append.OnNext(message);
}
_hartbeat.Subscribe(SendHartbeat);
}
