public async Task <SendLeaderCommandResponse> Receive(ICommand command)
{
if (State is Follower)
{
await _messageBus.Send(command, LeaderId);
}
if (State is Leader)
{
_logger.LogDebug("Server Received Command");
_appendingEntries = true;
Log.Add(new Log(CurrentTerm, command));
CommitIndex = Log.Count - 1;
var remoteServers = GetRemoteServers();
var tasks = new Task <AppendEntriesResponse> [remoteServers.Count];
for (int i = 0; i < tasks.Length; i++)
{
var next = NextIndex.FirstOrDefault(x => x.Id == remoteServers[i].Id);
if (next == null)
{
var nextLogIndex = 0;
next = new Next(remoteServers[i].Id, nextLogIndex);
NextIndex.Add(next);
}
var match = MatchIndex.FirstOrDefault(x => x.Id == remoteServers[i].Id);
if (match == null)
{
match = new Match(remoteServers[i].Id, 0);
MatchIndex.Add(match);
}
var lastLogIndex = Log.Count > 0 ? Log.Count - 1 : 0;
var lastLogTerm = lastLogIndex > 0 ? Log[match.MatchIndex].Term : 0;
// If last log index ≥ nextIndex for a follower: send
// AppendEntries RPC with log entries starting at nextIndex
if (lastLogIndex >= next.NextIndex)
{
var log = Log[next.NextIndex];
var appendEntries = new AppendEntries(CurrentTerm, Id, match.MatchIndex, lastLogTerm, log, CommitIndex, remoteServers[i].Id);
tasks[i] = _messageBus.Send(appendEntries);
}
}
Task.WaitAll(tasks);
int counter = 0;
foreach (var task in tasks)
{
_logger.LogDebug($"Processing Append entries counter: {counter}");
_logger.LogDebug($"Processing Append entries result was: {task.Result.Success} counter: {counter}");
await Receive(task.Result);
_logger.LogDebug($"Processed Append entries counter: {counter}");
}
}
return(new SendLeaderCommandResponse());
}
private void ResetLeaderState()
{
NextIndex.Clear();
MatchIndex.Clear();
Cluster.GetNodeIdsExcept(NodeId).ForEach(x => {
NextIndex[x] = Log.Count;
MatchIndex[x] = 0;
});
}
/// <summary>
/// Triggered by the heartbeatTimer. Sends AppendEntries requests in parallel to all the nodes in the cluster.
/// If there are unreplicated entries, sends them in the request. Sends a simple heartbeat otherwise.
/// It also checks MatchIndex for any entries replicated in the majority of nodes, and commits them.
/// </summary>
/// <param name="arg">Sent by System.Threading.Timer</param>
private void SendHeartbeats(object arg)
{
var nodes = Cluster.GetNodeIdsExcept(NodeId);
Parallel.ForEach(nodes, nodeId =>
{
if (!NextIndex.ContainsKey(nodeId))
{
return; // Prevents errors when testing
}
var prevLogIndex = Math.Max(0, NextIndex[nodeId] - 1);
int prevLogTerm = (Log.Count > 0) ? prevLogTerm = Log[prevLogIndex].TermNumber : 0;
List <LogEntry> entries;
if (Log.Count > NextIndex[nodeId])
{
LogMessage("Log Count: " + Log.Count + " -- Target node[nextIndex]: " + nodeId + " [" + NextIndex[nodeId] + "]");
entries = Log.Skip(NextIndex[nodeId]).ToList();
}
else
{
// covers Log is empty or no new entries to replicate
entries = null;
}
var res = Cluster.SendAppendEntriesTo(nodeId, CurrentTerm, NodeId, prevLogIndex, prevLogTerm, entries, CommitIndex);
CurrentTerm = res.Term;
if (res.Value)
{
if (entries != null)
{
// Entry appended
LogMessage("Successful AE to " + nodeId + ". Setting nextIndex to " + NextIndex[nodeId]);
NextIndex[nodeId] = Log.Count;
MatchIndex[nodeId] = Log.Count - 1;
}
}
else
{
LogMessage("Failed AE to " + nodeId + ". Setting nextIndex to " + NextIndex[nodeId]);
// Entry failed to be appended
if (NextIndex[nodeId] > 0)
{
NextIndex[nodeId]--;
}
}
});
// TODO: Do this as new task?
// Iterate over all uncommitted entries
for (int i = CommitIndex + 1; i < Log.Count; i++)
{
// We add 1 because we know the entry is replicated in this node
var replicatedIn = MatchIndex.Values.Count(x => x >= i) + 1;
if (Log[i].TermNumber == CurrentTerm && replicatedIn >= GetMajority())
{
CommitIndex = i;
StateMachine.Apply(Log[i].Command);
LastApplied = i;
}
}
// (responder a client request)
}
public void ResetLeaderState()
{
NextIndex.Clear();
MatchIndex.Clear();
}
private async Task Receive(AppendEntriesResponse appendEntriesResponse)
{
if (State is Leader)
{
// If RPC request or response contains term T > currentTerm:
// set currentTerm = T, convert to follower (§5.1)
if (appendEntriesResponse.Term > CurrentTerm)
{
BecomeFollowerAndMatchTerm(appendEntriesResponse.Term, appendEntriesResponse.FollowerId);
}
// If successful: update nextIndex and matchIndex for
// follower (§5.3)
if (State is Leader && appendEntriesResponse.Success)
{
var currentNext = NextIndex.First(x => x.Id == appendEntriesResponse.FollowerId);
NextIndex.Remove(currentNext);
var nextLogIndex = Log.Count;
var next = new Next(appendEntriesResponse.FollowerId, nextLogIndex);
NextIndex.Add(next);
var currentMatch = MatchIndex.First(x => x.Id == appendEntriesResponse.FollowerId);
MatchIndex.Remove(currentMatch);
var match = new Match(appendEntriesResponse.FollowerId, currentNext.NextIndex);
MatchIndex.Add(match);
}
if (State is Leader && _appendingEntries && appendEntriesResponse.Success)
{
CurrentTermAppendEntriesResponse++;
if (CurrentTermAppendEntriesResponse >= (_serversInClusterInCluster.Count / 2) + 1)
{
if ((CommitIndex == 0 && LastApplied == 0) || CommitIndex > LastApplied)
{
var entry = Log[Log.Count - 1];
LastApplied = Log.Count - 1;
await _stateMachine.Apply(entry.Command);
}
CurrentTermAppendEntriesResponse = 0;
_appendingEntries = false;
}
}
// If AppendEntries fails because of log inconsistency:
// decrement nextIndex and retry (§5.3)
else if (State is Leader && _appendingEntries)
{
var lastLogIndex = Log.Count - 1;
var lastLogTerm = Log[lastLogIndex].Term;
var next = NextIndex.First(x => x.Id == appendEntriesResponse.FollowerId);
if (lastLogIndex >= next.NextIndex)
{
var log = Log[next.NextIndex];
var appendEntries = new AppendEntries(CurrentTerm, Id, lastLogIndex, lastLogTerm, log, CommitIndex, appendEntriesResponse.FollowerId);
var task = _messageBus.Send(appendEntries);
Task.WaitAll(task);
await Receive(task.Result);
}
}
}
}
public PeerState(IPeer peer, MatchIndex matchIndex, NextIndex nextIndex)
{
Peer = peer;
MatchIndex = matchIndex;
NextIndex = nextIndex;
}
public void UpdateNextIndex(int nextLogIndexToSendToPeer)
{
NextIndex = new NextIndex(Peer, nextLogIndexToSendToPeer);
}