private async Task HeartBeatSend(CancellationToken c) { if (_role != Role.Leader) { return; } if (_lastHeartbeatSent.Since() < _settings.ElectionTimeoutMin.Multiply(0.2)) { return; } var currentTerm = State.CurrentTerm; // create a var. Could change during the method leading to confusing logs. var req = new AppendEntriesRequest() { CurrentTerm = currentTerm, Entries = new byte[0][], LeaderCommitIndex = _volatileState.CommitIndex, LeaderId = State.Id, PreviousLogIndex = long.MaxValue, PreviousLogTerm = long.MaxValue }; var peers = _peerManager.GetPeers().ToArray(); var proxies = peers.Select(x => _peerManager.GetProxy(x.Address)); var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(); var policy = Policy.TimeoutAsync(_settings.ElectionTimeoutMin.Multiply(0.2)).WrapAsync(retry); var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.AppendEntriesAsync(req)))); var maxTerm = currentTerm; foreach (var r in all) { if (r.Outcome == OutcomeType.Successful) { if (!r.Result.IsSuccess) { TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Got this reason for unsuccessful AppendEntriesAsync from a peer: {r.Result.Reason}"); } // NOTE: We do NOT change leadership if they send higher term, since they could be candidates whom will not become leaders // we actually do not need to do anything with the result other than logging it if (r.Result.CurrentTerm > maxTerm) { maxTerm = r.Result.CurrentTerm; } } } if (maxTerm > State.CurrentTerm) { TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Revolution brewing. Terms as high as {maxTerm} (vs my {currentTerm}) were seen."); } _lastHeartbeatSent.Set(); }
/// <inheritdoc /> public Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request) { _lastHeartbeat.Set(); if (request.Entries != null && request.Entries.Length > 0) { TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Received the dough {request.Entries.Length} for position after {request.PreviousLogIndex}"); } string message = null; lock (State) { if (request.CurrentTerm > State.CurrentTerm) { BecomeFollower(request.CurrentTerm); } } // Reply false if term < currentTerm (§5.1) if (request.CurrentTerm < State.CurrentTerm) { message = $"[{_meAsAPeer.ShortName}] Leader's term is behind ({request.CurrentTerm} vs {State.CurrentTerm})."; TheTrace.TraceWarning(message); return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.TermInconsistency, message))); } if (request.Entries == null || request.Entries.Length == 0) // it is a heartbeat, set the leader address { _leaderAddress = _peerManager.GetPeers().Where(x => x.Id == request.LeaderId).FirstOrDefault()?.Address; return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true))); } // chaos only when has entries _chaos.WreakHavoc(); if (request.PreviousLogIndex > _logPersister.LastIndex) { message = $"[{_meAsAPeer.ShortName}] Position for last log entry is {_logPersister.LastIndex} but got entries starting at {request.PreviousLogIndex}"; TheTrace.TraceWarning(message); return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message))); } if (request.PreviousLogIndex < _logPersister.LastIndex) { TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Position for PreviousLogIndex {request.PreviousLogIndex} but my LastIndex {_logPersister.LastIndex}"); // Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm(§5.3) var entry = _logPersister.GetEntries(request.PreviousLogIndex, 1).First(); if (entry.Term != request.CurrentTerm) { message = $"[{_meAsAPeer.ShortName}] Position at {request.PreviousLogIndex} has term {entry.Term} but according to leader {request.LeaderId} it must be {request.PreviousLogTerm}"; TheTrace.TraceWarning(message); return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message))); } // If an existing entry conflicts with a new one(same index but different terms), delete the existing entry and all that follow it(§5.3) _logPersister.DeleteEntries(request.PreviousLogIndex + 1); TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Stripping the log from index {request.PreviousLogIndex + 1}. Last index was {_logPersister.LastIndex}"); } var entries = request.Entries.Select(x => new LogEntry() { Body = x, Term = request.CurrentTerm }).ToArray(); // Append any new entries not already in the log TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Current last index is {_logPersister.LastIndex}. About to append {entries.Length} entries at {request.PreviousLogIndex + 1}"); _logPersister.Append(entries, request.PreviousLogIndex + 1); //If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry) if (request.LeaderCommitIndex > _volatileState.CommitIndex) { _volatileState.CommitIndex = Math.Min(request.LeaderCommitIndex, _logPersister.LastIndex); } message = $"[{_meAsAPeer.ShortName}] Appended {request.Entries.Length} entries at position {request.PreviousLogIndex + 1}"; TheTrace.TraceInformation(message); return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true, ReasonType.None, message))); }
private async Task SendLogs( IRaftServer proxy, AsyncPolicy policy, Peer peer, long nextIndex, long matchIndex, int count) { var previousIndexTerm = -1L; if (nextIndex > 0) { if (nextIndex > _logPersister.LogOffset) { previousIndexTerm = _logPersister.GetEntries(nextIndex - 1, 1).First().Term; } else { Snapshot ss; if (_snapshotOperator.TryGetLastSnapshot(out ss)) { previousIndexTerm = ss.LastIncludedTerm; } } } var request = new AppendEntriesRequest() { CurrentTerm = State.CurrentTerm, Entries = _logPersister.GetEntries(nextIndex, count).Select(x => x.Body).ToArray(), LeaderCommitIndex = _volatileState.CommitIndex, LeaderId = State.Id, PreviousLogIndex = nextIndex - 1, PreviousLogTerm = previousIndexTerm }; var result = await policy.ExecuteAndCaptureAsync(() => proxy.AppendEntriesAsync(request)); if (result.Outcome == OutcomeType.Successful) { if (result.Result.IsSuccess) { // If successful: update nextIndex and matchIndex for follower(§5.3)" _volatileLeaderState.SetMatchIndex(peer.Id, nextIndex + count - 1); _volatileLeaderState.SetNextIndex(peer.Id, nextIndex + count); TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Successfully transferred {count} entries from index {nextIndex} to peer {peer.Address} - Next Index is {_volatileLeaderState.NextIndices[peer.Id]}"); UpdateCommitIndex(); } else { // log reason only TheTrace.TraceWarning($"AppendEntries for start index {nextIndex} and count {count} for peer {peer.Address} with address {peer.Address} in term {State.CurrentTerm} failed with reason type {result.Result.ReasonType} and this reason: {result.Result.Reason}"); if (result.Result.ReasonType == ReasonType.LogInconsistency) { var diff = nextIndex - (matchIndex + 1); nextIndex = diff > _settings.MaxNumberOfDecrementForLogsThatAreBehind ? nextIndex - _settings.MaxNumberOfDecrementForLogsThatAreBehind : nextIndex - diff; _volatileLeaderState.SetNextIndex(peer.Id, nextIndex); TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Updated (decremented) next index for peer {peer.Address} to {nextIndex}"); } } } else { // NUNCA!! // not interested in network, etc errors, they get logged in the policy } }