/// <summary>
/// This method may run for a long while, as we are trying to get agreement
/// from a majority of the cluster
/// </summary>
private void Run()
{
try
{
while (_candidate.Running && _disposed == false)
{
try
{
Stream stream;
try
{
using (_engine.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
stream = _engine.ConnectToPeer(_url, _certificate, context).Result;
}
if (_candidate.Running == false)
{
break;
}
}
catch (Exception e)
{
Status = AmbassadorStatus.FailedToConnect;
StatusMessage = $"Failed to connect with {_tag}.{Environment.NewLine} " + e.Message;
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: Failed to connect to remote peer: " + _url, e);
}
// wait a bit
_candidate.WaitForChangeInState();
continue; // we'll retry connecting
}
Status = AmbassadorStatus.Connected;
StatusMessage = $"Connected to {_tag}";
Connection = new RemoteConnection(_tag, _engine.Tag, stream);
using (_engine.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
ClusterTopology topology;
long lastLogIndex;
long lastLogTerm;
using (context.OpenReadTransaction())
{
topology = _engine.GetTopology(context);
lastLogIndex = _engine.GetLastEntryIndex(context);
lastLogTerm = _engine.GetTermForKnownExisting(context, lastLogIndex);
}
Debug.Assert(topology.TopologyId != null);
Connection.Send(context, new RachisHello
{
TopologyId = topology.TopologyId,
DebugSourceIdentifier = _engine.Tag,
DebugDestinationIdentifier = _tag,
InitialMessageType = InitialMessageType.RequestVote
});
while (_candidate.Running)
{
RequestVoteResponse rvr;
var currentElectionTerm = _candidate.ElectionTerm;
var engineCurrentTerm = _engine.CurrentTerm;
if (_candidate.IsForcedElection == false ||
_candidate.RunRealElectionAtTerm != currentElectionTerm)
{
Connection.Send(context, new RequestVote
{
Source = _engine.Tag,
Term = currentElectionTerm,
IsForcedElection = false,
IsTrialElection = true,
LastLogIndex = lastLogIndex,
LastLogTerm = lastLogTerm
});
rvr = Connection.Read <RequestVoteResponse>(context);
if (rvr.Term > currentElectionTerm)
{
var message = $"Candidate ambassador {_engine.Tag}: found election term {rvr.Term} that is higher than ours {currentElectionTerm}";
// we need to abort the current elections
_engine.SetNewState(RachisState.Follower, null, engineCurrentTerm, message);
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: {message}");
}
_engine.FoundAboutHigherTerm(rvr.Term);
throw new InvalidOperationException(message);
}
NotInTopology = rvr.NotInTopology;
if (rvr.VoteGranted == false)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: Got a negative response from {_tag} reason: {rvr.Message}");
}
// we go a negative response here, so we can't proceed
// we'll need to wait until the candidate has done something, like
// change term or given up
_candidate.WaitForChangeInState();
continue;
}
TrialElectionWonAtTerm = rvr.Term;
_candidate.WaitForChangeInState();
}
Connection.Send(context, new RequestVote
{
Source = _engine.Tag,
Term = currentElectionTerm,
IsForcedElection = _candidate.IsForcedElection,
IsTrialElection = false,
LastLogIndex = lastLogIndex,
LastLogTerm = lastLogTerm
});
rvr = Connection.Read <RequestVoteResponse>(context);
if (rvr.Term > currentElectionTerm)
{
var message = $"Candidate ambassador {_engine.Tag}: found election term {rvr.Term} that is higher than ours {currentElectionTerm}";
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: {message}");
}
// we need to abort the current elections
_engine.SetNewState(RachisState.Follower, null, engineCurrentTerm, message);
_engine.FoundAboutHigherTerm(rvr.Term);
throw new InvalidOperationException(message);
}
NotInTopology = rvr.NotInTopology;
if (rvr.VoteGranted == false)
{
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: Got a negative response from {_tag} reason: {rvr.Message}");
}
// we go a negative response here, so we can't proceed
// we'll need to wait until the candidate has done something, like
// change term or given up
_candidate.WaitForChangeInState();
continue;
}
RealElectionWonAtTerm = rvr.Term;
_candidate.WaitForChangeInState();
}
SendElectionResult();
}
}
catch (OperationCanceledException)
{
Status = AmbassadorStatus.Closed;
StatusMessage = "Closed";
SendElectionResult();
break;
}
catch (ObjectDisposedException)
{
Status = AmbassadorStatus.Closed;
StatusMessage = "Closed";
SendElectionResult();
break;
}
catch (AggregateException ae)
when(ae.InnerException is OperationCanceledException || ae.InnerException is ObjectDisposedException)
{
Status = AmbassadorStatus.Closed;
StatusMessage = "Closed";
SendElectionResult();
break;
}
catch (Exception e)
{
Status = AmbassadorStatus.FailedToConnect;
StatusMessage = $"Failed to get vote from {_tag}.{Environment.NewLine}" + e.Message;
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info($"CandidateAmbassador {_engine.Tag}: Failed to get vote from remote peer url={_url} tag={_tag}", e);
}
Connection?.Dispose();
_candidate.WaitForChangeInState();
}
}
}
catch (Exception e)
{
Status = AmbassadorStatus.FailedToConnect;
StatusMessage = $"Failed to talk to {_url}.{Environment.NewLine}" + e;
if (_engine.Log.IsInfoEnabled)
{
_engine.Log.Info("Failed to talk to remote peer: " + _url, e);
}
}
finally
{
if (_candidate.ElectionResult != ElectionResult.Won)
{
Connection?.Dispose();
}
}
}
