private async Task SendLogs(
IRaftServer proxy,
AsyncPolicy policy,
Peer peer,
long nextIndex,
long matchIndex,
int count)
{
var previousIndexTerm = -1L;
if (nextIndex > 0)
{
if (nextIndex > _logPersister.LogOffset)
{
previousIndexTerm = _logPersister.GetEntries(nextIndex - 1, 1).First().Term;
}
else
{
Snapshot ss;
if (_snapshotOperator.TryGetLastSnapshot(out ss))
{
previousIndexTerm = ss.LastIncludedTerm;
}
}
}
var request = new AppendEntriesRequest()
{
CurrentTerm = State.CurrentTerm,
Entries = _logPersister.GetEntries(nextIndex, count).Select(x => x.Body).ToArray(),
LeaderCommitIndex = _volatileState.CommitIndex,
LeaderId = State.Id,
PreviousLogIndex = nextIndex - 1,
PreviousLogTerm = previousIndexTerm
};
var result = await policy.ExecuteAndCaptureAsync(() => proxy.AppendEntriesAsync(request));
if (result.Outcome == OutcomeType.Successful)
{
if (result.Result.IsSuccess)
{
// If successful: update nextIndex and matchIndex for follower(§5.3)"
_volatileLeaderState.SetMatchIndex(peer.Id, nextIndex + count - 1);
_volatileLeaderState.SetNextIndex(peer.Id, nextIndex + count);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Successfully transferred {count} entries from index {nextIndex} to peer {peer.Address} - Next Index is {_volatileLeaderState.NextIndices[peer.Id]}");
UpdateCommitIndex();
}
else
{
// log reason only
TheTrace.TraceWarning($"AppendEntries for start index {nextIndex} and count {count} for peer {peer.Address} with address {peer.Address} in term {State.CurrentTerm} failed with reason type {result.Result.ReasonType} and this reason: {result.Result.Reason}");
if (result.Result.ReasonType == ReasonType.LogInconsistency)
{
var diff = nextIndex - (matchIndex + 1);
nextIndex = diff > _settings.MaxNumberOfDecrementForLogsThatAreBehind ?
nextIndex - _settings.MaxNumberOfDecrementForLogsThatAreBehind :
nextIndex - diff;
_volatileLeaderState.SetNextIndex(peer.Id, nextIndex);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Updated (decremented) next index for peer {peer.Address} to {nextIndex}");
}
}
}
else
{
// NUNCA!!
// not interested in network, etc errors, they get logged in the policy
}
}
private async Task SendSnapshot(
IRaftServer proxy,
AsyncPolicy policy,
Peer peer,
long nextIndex,
long matchIndex)
{
var logOffset = LogPersister.LogOffset;
var term = State.CurrentTerm;
Snapshot ss;
if (!SnapshotOperator.TryGetLastSnapshot(out ss))
{
throw new InvalidProgramException($"WE DO NOT HAVE A SNAPSHOT for client {peer.Address} whose nextIndex is {nextIndex} yet our LogOffset is {logOffset}");
}
if (ss.LastIncludedIndex + 1 < nextIndex)
{
throw new InvalidProgramException($"WE DO NOT HAVE A <<PROPER>> SNAPSHOT for client {peer.Address} whose nextIndex is {nextIndex} yet our LogOffset is {logOffset}. Snapshot was have ({ss.FullName}) is short {ss.LastIncludedIndex}");
}
// make a copy since it might be cleaned up or opened by another thread for another client
var fileName = Path.GetTempFileName();
File.Copy(ss.FullName, fileName, true);
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] About to send Snapshot copy file {fileName} to [{peer.ShortName}] and copy of {ss.FullName}.");
using (var fs = new FileStream(fileName, FileMode.Open))
{
var start = 0;
var total = 0;
var length = fs.Length;
var buffer = new byte[_settings.MaxSnapshotChunkSentInBytes];
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Snapshot copy file size is {length}. Location is {fileName} and copy of {ss.FullName}.");
while (total < length)
{
var count = fs.Read(buffer, 0, buffer.Length);
total += count;
var result = await proxy.InstallSnapshotAsync(new InstallSnapshotRequest()
{
CurrentTerm = term,
Data = count == buffer.Length ? buffer : buffer.Take(count).ToArray(),
LastIncludedIndex = ss.LastIncludedIndex,
LastIncludedTerm = term,
IsDone = total == length,
LeaderId = State.Id,
Offset = start
});
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Sent snapshot for peer {peer.Address} with {count} bytes totalling {total}.");
start += count;
if (result.CurrentTerm != term)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] I am sending snapshot but this peer {peer.Address} has term {result.CurrentTerm} vs my started term {term} and current term {State.CurrentTerm}.");
}
}
}
_volatileLeaderState.SetMatchIndex(peer.Id, ss.LastIncludedIndex);
_volatileLeaderState.SetNextIndex(peer.Id, ss.LastIncludedIndex + 1); // the rest will be done by sending logs
File.Delete(fileName);
}
