public void RaciveHartbeat(AppendEntryMessage appendEntry) { if (_serverIdentifier.Equals(appendEntry.Leader)) return; if (_options.UseLogging) _logger.LogInformation($"Processing {nameof(RaciveHartbeat)} \n\t\t Leader {appendEntry.Leader} \n\t\t Term {appendEntry.Term}"); // If not in your term reject request if (appendEntry.Term < _election.CurrentTerm) { _entryReply.OnNext(new AppendEntryResultMessage() { Term = _election.CurrentTerm, Success = false, }); return; } // Update to follow new term if (appendEntry.Term > _election.CurrentTerm) { _election.CurrentTerm = appendEntry.Term; _election.VotedFor = appendEntry.Leader; _state?.OnNext(ServerStateType.Follower); _entryReply.OnNext(new AppendEntryResultMessage() { Term = _election.CurrentTerm, Success = false, }); return; } _hartbeat.Reset(); var term = _election.CurrentTerm; var outOfSync = false; if (_logReplication.Contains(appendEntry.PreviousLogIndex)) { // If the appendEntry from the leader is in sync with us, the follower if (_logReplication.Term(appendEntry.PreviousLogIndex) == appendEntry.PreviousLogTerm) { // If we need to rollback the logs var lastApplied = appendEntry.PreviousLogIndex + 1; if (lastApplied != _logReplication.LastApplied) { _logReplication.Rollback(lastApplied); } // We can start processing the logs var successes = new List<bool>(); if (appendEntry.Entries != null) { foreach (var log in appendEntry.Entries) { successes.Add(_logReplication.Append(log)); } } // Confirm Log append _entryReply.OnNext(new AppendEntryResultMessage() { Term = term, LogIndex = _logReplication.LastApplied, LogTerm = _logReplication.LastTerm, Success = successes.TrueForAll(p => p), From = _serverIdentifier }); } else { outOfSync = true; } } else { outOfSync = true; } // reject the append entry as the leader is out of sync with the follower if (outOfSync) { // Will keep moving the log index back until the followr and the leader match on the log consistency // This will let the lead overwrite the followers logs to bring it back into full consistency. var previousIndex = _logReplication.LastApplied; if (appendEntry.PreviousLogIndex <= previousIndex) { previousIndex = appendEntry.PreviousLogIndex - 1; } var previousTerm = _logReplication.Term(previousIndex); // update the lead with our last index and term of a successfully append _entryReply.OnNext(new AppendEntryResultMessage() { Term = term, LogIndex = previousIndex, LogTerm = previousTerm, Success = false, From = _serverIdentifier }); } var logsToProcess = _logReplication.ToCommit(appendEntry.LeaderCommit, appendEntry.Term); // Process the logs to commit as a task so not to block RPC ProcessLogs(logsToProcess); }
public void SendHartbeat(ServerStateType type) { if (_isDispose) return; var term = _election.CurrentTerm; var votedFor = _election.VotedFor; foreach (var node in _nodes) { if (_isDispose) return; if (!NextIndex.ContainsKey(node)) { // Adds a node to our record and assumes it's log state is the same as our's // If it's not the same the follower will reject the message and send // back it's current state for the next heartbeat NextIndex.TryAdd(node, _logReplication.LastApplied); MatchIndex.TryAdd(node, 0); } var next = NextIndex[node]; var previousLogIndex = next - 1; var entries = _logReplication.Replicate(next); var logEntries = entries as IList<LogEntry> ?? entries.ToList(); var message = new AppendEntryMessage() { To = node, Term = term, Leader = votedFor, Entries = logEntries, LeaderCommit = _logReplication.CommitIndex, PreviousLogIndex = previousLogIndex, PreviousLogTerm = _logReplication.Term(previousLogIndex) }; if (_options.UseLogging) _logger.LogInformation($"Sending {nameof(AppendEntryMessage)} \n\t\t Term {message.Term} \n\t\t To {message.To} \n\t\t Leader {message.Leader} \n\t\t Entries {logEntries.Count()}"); _append.OnNext(message); } _hartbeat.Subscribe(SendHartbeat); }