private void NegotiateWithLeader(ClusterOperationContext context, LogLengthNegotiation negotiation)
{
_debugRecorder.Start();
// only the leader can send append entries, so if we accepted it, it's the leader
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request for term {negotiation.Term} where our term is {_term}.");
}
if (negotiation.Term != _term)
{
// Our leader is no longer a valid one
var msg = $"The term was changed after leader validation from {_term} to {negotiation.Term}, so you are no longer a valid leader";
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Rejected,
Message = msg,
CurrentTerm = _term,
LastLogIndex = negotiation.PrevLogIndex
});
throw new InvalidOperationException($"We close this follower because: {msg}");
}
long prevTerm;
bool requestSnapshot;
bool requestFullSnapshot = false;
using (context.OpenReadTransaction())
{
prevTerm = _engine.GetTermFor(context, negotiation.PrevLogIndex) ?? 0;
requestSnapshot = _engine.GetSnapshotRequest(context);
}
if (requestSnapshot)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Request snapshot by the admin");
}
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = "Request snapshot by the admin",
CurrentTerm = _term,
LastLogIndex = 0
});
requestFullSnapshot = true;
}
else if (prevTerm != negotiation.PrevLogTerm)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request with PrevLogTerm={negotiation.PrevLogTerm} while our PrevLogTerm={prevTerm}" +
" will negotiate to find next matched index");
}
// we now have a mismatch with the log position, and need to negotiate it with
// the leader
requestFullSnapshot = NegotiateMatchEntryWithLeaderAndApplyEntries(context, _connection, negotiation);
}
else
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"{ToString()}: Got a negotiation request with identical PrevLogTerm will continue to steady state");
}
// this (or the negotiation above) completes the negotiation process
_connection.Send(context, new LogLengthNegotiationResponse
{
Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable,
Message = $"Found a log index / term match at {negotiation.PrevLogIndex} with term {prevTerm}",
CurrentTerm = _term,
LastLogIndex = negotiation.PrevLogIndex
});
}
_debugRecorder.Record("Matching Negotiation is over, waiting for snapshot");
_engine.Timeout.Defer(_connection.Source);
// at this point, the leader will send us a snapshot message
// in most cases, it is an empty snapshot, then start regular append entries
// the reason we send this is to simplify the # of states in the protocol
var snapshot = _connection.ReadInstallSnapshot(context);
_debugRecorder.Record("Start receiving snapshot");
// reading the snapshot from network and committing it to the disk might take a long time.
using (var cts = new CancellationTokenSource())
{
KeepAliveAndExecuteAction(() => ReadAndCommitSnapshot(context, snapshot, cts.Token), cts, "ReadAndCommitSnapshot");
}
_debugRecorder.Record("Snapshot was received and committed");
// notify the state machine, we do this in an async manner, and start
// the operator in a separate thread to avoid timeouts while this is
// going on
using (var cts = new CancellationTokenSource())
{
KeepAliveAndExecuteAction(() =>
{
_engine.SnapshotInstalled(snapshot.LastIncludedIndex, requestFullSnapshot, cts.Token);
}, cts, "SnapshotInstalledAsync");
}
_debugRecorder.Record("Done with StateMachine.SnapshotInstalled");
_debugRecorder.Record("Snapshot installed");
//Here we send the LastIncludedIndex as our matched index even for the case where our lastCommitIndex is greater
//So we could validate that the entries sent by the leader are indeed the same as the ones we have.
_connection.Send(context, new InstallSnapshotResponse
{
Done = true,
CurrentTerm = _term,
LastLogIndex = snapshot.LastIncludedIndex
});
_engine.Timeout.Defer(_connection.Source);
}
