private async Task HeartBeatSend(CancellationToken c)
{
if (_role != Role.Leader)
{
return;
}
if (_lastHeartbeatSent.Since() < _settings.ElectionTimeoutMin.Multiply(0.2))
{
return;
}
var currentTerm = State.CurrentTerm; // create a var. Could change during the method leading to confusing logs.
var req = new AppendEntriesRequest()
{
CurrentTerm = currentTerm,
Entries = new byte[0][],
LeaderCommitIndex = _volatileState.CommitIndex,
LeaderId = State.Id,
PreviousLogIndex = long.MaxValue,
PreviousLogTerm = long.MaxValue
};
var peers = _peerManager.GetPeers().ToArray();
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync();
var policy = Policy.TimeoutAsync(_settings.ElectionTimeoutMin.Multiply(0.2)).WrapAsync(retry);
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.AppendEntriesAsync(req))));
var maxTerm = currentTerm;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (!r.Result.IsSuccess)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Got this reason for unsuccessful AppendEntriesAsync from a peer: {r.Result.Reason}");
}
// NOTE: We do NOT change leadership if they send higher term, since they could be candidates whom will not become leaders
// we actually do not need to do anything with the result other than logging it
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
}
}
if (maxTerm > State.CurrentTerm)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Revolution brewing. Terms as high as {maxTerm} (vs my {currentTerm}) were seen.");
}
_lastHeartbeatSent.Set();
}
/// <inheritdoc />
public Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request)
{
_lastHeartbeat.Set();
if (request.Entries != null && request.Entries.Length > 0)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Received the dough {request.Entries.Length} for position after {request.PreviousLogIndex}");
}
string message = null;
lock (State)
{
if (request.CurrentTerm > State.CurrentTerm)
{
BecomeFollower(request.CurrentTerm);
}
}
// Reply false if term < currentTerm (§5.1)
if (request.CurrentTerm < State.CurrentTerm)
{
message = $"[{_meAsAPeer.ShortName}] Leader's term is behind ({request.CurrentTerm} vs {State.CurrentTerm}).";
TheTrace.TraceWarning(message);
return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.TermInconsistency, message)));
}
if (request.Entries == null || request.Entries.Length == 0) // it is a heartbeat, set the leader address
{
_leaderAddress = _peerManager.GetPeers().Where(x => x.Id == request.LeaderId).FirstOrDefault()?.Address;
return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true)));
}
// chaos only when has entries
_chaos.WreakHavoc();
if (request.PreviousLogIndex > _logPersister.LastIndex)
{
message = $"[{_meAsAPeer.ShortName}] Position for last log entry is {_logPersister.LastIndex} but got entries starting at {request.PreviousLogIndex}";
TheTrace.TraceWarning(message);
return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message)));
}
if (request.PreviousLogIndex < _logPersister.LastIndex)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Position for PreviousLogIndex {request.PreviousLogIndex} but my LastIndex {_logPersister.LastIndex}");
// Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm(§5.3)
var entry = _logPersister.GetEntries(request.PreviousLogIndex, 1).First();
if (entry.Term != request.CurrentTerm)
{
message = $"[{_meAsAPeer.ShortName}] Position at {request.PreviousLogIndex} has term {entry.Term} but according to leader {request.LeaderId} it must be {request.PreviousLogTerm}";
TheTrace.TraceWarning(message);
return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message)));
}
// If an existing entry conflicts with a new one(same index but different terms), delete the existing entry and all that follow it(§5.3)
_logPersister.DeleteEntries(request.PreviousLogIndex + 1);
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Stripping the log from index {request.PreviousLogIndex + 1}. Last index was {_logPersister.LastIndex}");
}
var entries = request.Entries.Select(x => new LogEntry()
{
Body = x,
Term = request.CurrentTerm
}).ToArray();
// Append any new entries not already in the log
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Current last index is {_logPersister.LastIndex}. About to append {entries.Length} entries at {request.PreviousLogIndex + 1}");
_logPersister.Append(entries, request.PreviousLogIndex + 1);
//If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry)
if (request.LeaderCommitIndex > _volatileState.CommitIndex)
{
_volatileState.CommitIndex = Math.Min(request.LeaderCommitIndex, _logPersister.LastIndex);
}
message = $"[{_meAsAPeer.ShortName}] Appended {request.Entries.Length} entries at position {request.PreviousLogIndex + 1}";
TheTrace.TraceInformation(message);
return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true, ReasonType.None, message)));
}
private async Task SendLogs(
IRaftServer proxy,
AsyncPolicy policy,
Peer peer,
long nextIndex,
long matchIndex,
int count)
{
var previousIndexTerm = -1L;
if (nextIndex > 0)
{
if (nextIndex > _logPersister.LogOffset)
{
previousIndexTerm = _logPersister.GetEntries(nextIndex - 1, 1).First().Term;
}
else
{
Snapshot ss;
if (_snapshotOperator.TryGetLastSnapshot(out ss))
{
previousIndexTerm = ss.LastIncludedTerm;
}
}
}
var request = new AppendEntriesRequest()
{
CurrentTerm = State.CurrentTerm,
Entries = _logPersister.GetEntries(nextIndex, count).Select(x => x.Body).ToArray(),
LeaderCommitIndex = _volatileState.CommitIndex,
LeaderId = State.Id,
PreviousLogIndex = nextIndex - 1,
PreviousLogTerm = previousIndexTerm
};
var result = await policy.ExecuteAndCaptureAsync(() => proxy.AppendEntriesAsync(request));
if (result.Outcome == OutcomeType.Successful)
{
if (result.Result.IsSuccess)
{
// If successful: update nextIndex and matchIndex for follower(§5.3)"
_volatileLeaderState.SetMatchIndex(peer.Id, nextIndex + count - 1);
_volatileLeaderState.SetNextIndex(peer.Id, nextIndex + count);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Successfully transferred {count} entries from index {nextIndex} to peer {peer.Address} - Next Index is {_volatileLeaderState.NextIndices[peer.Id]}");
UpdateCommitIndex();
}
else
{
// log reason only
TheTrace.TraceWarning($"AppendEntries for start index {nextIndex} and count {count} for peer {peer.Address} with address {peer.Address} in term {State.CurrentTerm} failed with reason type {result.Result.ReasonType} and this reason: {result.Result.Reason}");
if (result.Result.ReasonType == ReasonType.LogInconsistency)
{
var diff = nextIndex - (matchIndex + 1);
nextIndex = diff > _settings.MaxNumberOfDecrementForLogsThatAreBehind ?
nextIndex - _settings.MaxNumberOfDecrementForLogsThatAreBehind :
nextIndex - diff;
_volatileLeaderState.SetNextIndex(peer.Id, nextIndex);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Updated (decremented) next index for peer {peer.Address} to {nextIndex}");
}
}
}
else
{
// NUNCA!!
// not interested in network, etc errors, they get logged in the policy
}
}