/// <summary>
/// Receive RequestVote RPC request
/// </summary>
public async Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
await _nodeLock.WaitAsync();
var response = new RequestVoteResponse(State.CurrentTerm, false, "Unknown error");
try
{
if (request.Term < State.CurrentTerm)
{
// 1. Reply false if term < currentTerm (§5.1)
response = new RequestVoteResponse(State.CurrentTerm, false, "term < currentTerm");
}
else
{
var lastTermAndIndex = await _log.GetLastTermAndIndexAsync();
if (request.Term > State.CurrentTerm)
{
// If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower (§5.1)
State.SetTerm(request.Term);
BecomeFollower();
}
if (Role == NodeRole.Follower &&
(string.IsNullOrEmpty(State.VotedFor) || State.VotedFor == request.CandidateId))
{
if ((request.LastLogTerm == lastTermAndIndex.term && request.LastLogIndex == lastTermAndIndex.index) ||
(request.LastLogTerm >= lastTermAndIndex.term && request.LastLogIndex > lastTermAndIndex.index))
{
// 2. If votedFor is null or candidateId, and candidate’s log is at
// least as up-to-date as receiver’s log, grant vote (§5.2, §5.4)
Interlocked.Exchange(ref _receivedAppendEntriesOrGrantedVote, 1);
State.VoteFor(request.CandidateId);
response = new RequestVoteResponse(State.CurrentTerm, true);
}
else
{
Console.WriteLine($"request.LastLogTerm={request.LastLogTerm}, request.LastLogIndex={request.LastLogIndex}");
Console.WriteLine($"lastTermAndIndex.term={lastTermAndIndex.term}, lastTermAndIndex.index={lastTermAndIndex.index}");
response = new RequestVoteResponse(State.CurrentTerm, false, "candidate’s log is not up-to-date as receiver’s log");
}
}
else
{
response = new RequestVoteResponse(State.CurrentTerm, false, "Not follower or has grant vote to another candicate");
}
}
return(response);
}
catch (Exception ex)
{
Console.WriteLine($"Handle RequestVoteAsync fail: {ex.Message}\n{ex.StackTrace}");
throw;
}
finally
{
_nodeLock.Release();
}
}
public Task <HttpResponseMessage> RequestVote([FromUri] RequestVoteRequest request)
{
var taskCompletionSource = new TaskCompletionSource <HttpResponseMessage>();
_bus.Publish(request, taskCompletionSource);
return(taskCompletionSource.Task);
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
if (MaybeIgnoreFrequentRequestsIfServerDown(dest, "vote"))
{
return;
}
LogStatus("request vote from " + dest, async() =>
{
var requestUri = string.Format("raft/requestVote?term={0}&lastLogIndex={1}&lastLogTerm={2}&trialOnly={3}&forcedElection={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LastLogIndex, req.LastLogTerm, req.TrialOnly, req.ForcedElection, req.From, req.ClusterTopologyId);
using (var request = CreateRequest(dest, _shortOperationsTimeout, requestUri, HttpMethods.Get, _log))
{
var httpResponseMessage = await request.ExecuteAsync().ConfigureAwait(false);
UpdateConnectionFailureCounts(dest, httpResponseMessage);
var reply = await httpResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error requesting vote from {0}. Status: {1}\r\n{2}\r\nreason:{3}", dest.Name, httpResponseMessage.StatusCode, reply, httpResponseMessage.ReasonPhrase);
return;
}
if (httpResponseMessage.StatusCode == HttpStatusCode.NotAcceptable)
{
_log.Warn("Error requesting vote from {0}. Status: NotAcceptable\r\nreason:{1}\r\ncontent{2}", dest.Name, httpResponseMessage.ReasonPhrase, reply);
}
var requestVoteResponse = JsonConvert.DeserializeObject <RequestVoteResponse>(reply);
SendToSelf(requestVoteResponse);
}
});
}
public async Task RequestsVotesOnEntry()
{
// Setup: configure grains to always grant votes
this.OnRaftGrainCreated = (id, grain) =>
{
var raftGrain = grain;
raftGrain?.RequestVote(Arg.Any <RequestVoteRequest>())
.Returns(Task.FromResult(new RequestVoteResponse {
Term = 1, VoteGranted = true
}));
};
await this.role.Enter();
foreach (var server in this.members.AllServers)
{
var grain = this.grainFactory.GetGrain <IRaftGrain <int> >(server);
if (string.Equals(server, this.identity.Id, StringComparison.Ordinal))
{
await grain.DidNotReceive().RequestVote(Arg.Any <RequestVoteRequest>());
continue;
}
var request = new RequestVoteRequest(
this.persistentState.CurrentTerm,
this.identity.Id,
this.journal.LastLogEntryId);
await grain.Received().RequestVote(request);
}
}
/// <summary>
///
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
try
{
var json = JsonConvert.SerializeObject(request);
var content = new StringContent(json);
content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
var response = await _httpClient.PostAsync("/_raft/requestvote", content);
if (response != null && response.IsSuccessStatusCode)
{
var responseContent = await response.Content.ReadAsStringAsync();
return(JsonConvert.DeserializeObject <RequestVoteResponse>(responseContent));
}
return(new RequestVoteResponse(request.Term, false, "Server Internal Error"));
}
catch (HttpRequestException requestEx)
{
_logger.LogError($"RequestVote fail: {requestEx.Message}, serverUri: {_httpClient.BaseAddress}");
return(new RequestVoteResponse(request.Term, false, requestEx.Message));
}
catch (Exception ex)
{
_logger.LogError(ex, $"RequestVote fail: {ex.Message}, serverUri: {_httpClient.BaseAddress}");
return(new RequestVoteResponse(request.Term, false, ex.Message));
}
}
private void StartElection()
{
LastHeartbeatTime = DateTime.UtcNow;
_votesForMyLeadership.Clear();
long currentTerm = Engine.PersistentState.CurrentTerm + 1;
if (_wonTrialElection || _termIncreaseMightGetMyVote) // only in the real election (or if we have to), we increment the current term
{
Engine.PersistentState.UpdateTermTo(Engine, currentTerm);
}
if (_wonTrialElection) // and only if we won an election do we record a firm vote for ourselves
{
Engine.PersistentState.RecordVoteFor(Engine.Name, currentTerm);
}
_termIncreaseMightGetMyVote = false;
Engine.CurrentLeader = null;
_log.Info("Calling for {0} election in term {1}",
_wonTrialElection ? "an" : "a trial", currentTerm);
var lastLogEntry = Engine.PersistentState.LastLogEntry();
var rvr = new RequestVoteRequest
{
LastLogIndex = lastLogEntry.Index,
LastLogTerm = lastLogEntry.Term,
Term = currentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = _wonTrialElection == false,
ForcedElection = _forcedElection
};
var allVotingNodes = Engine.CurrentTopology.AllVotingNodes;
// don't send to yourself the message
foreach (var votingPeer in allVotingNodes)
{
if (votingPeer.Name == Engine.Name)
{
continue;
}
Engine.Transport.Send(votingPeer, rvr);
}
Engine.OnCandidacyAnnounced();
_log.Info("Voting for myself in {1}election for term {0}", currentTerm, _wonTrialElection ? " " : " trial ");
Handle(new RequestVoteResponse
{
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteGranted = true,
Message = String.Format("{0} -> Voting for myself", Engine.Name),
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
VoteTerm = currentTerm,
TrialOnly = _wonTrialElection == false
});
}
public async Task <RequestVoteResponse> RequestVote(RequestVoteRequest request)
{
bool voteGranted;
this.logger.LogInfo($"RequestVote: {request}");
// 1. Reply false if term < currentTerm(§5.1)
if (request.Term < this.persistentState.CurrentTerm)
{
this.logger.LogWarn($"Denying vote {request}. Requested term is older than current term.");
voteGranted = false;
}
// 2. If votedFor is null or candidateId, and candidate’s log is at least as up-to-date as receiver’s
// log, grant vote (§5.2, §5.4)
else
{
// Check if this server has already voted for another server in the current term.
var votedInCurrentTerm = request.Term == this.persistentState.CurrentTerm &&
!string.IsNullOrEmpty(this.persistentState.VotedFor);
var votedForAnotherServerInCurrentTerm = votedInCurrentTerm &&
!string.Equals(
this.persistentState.VotedFor,
request.Candidate,
StringComparison.Ordinal);
if (votedForAnotherServerInCurrentTerm)
{
this.logger.LogWarn($"Denying vote {request}: Already voted for {this.persistentState.VotedFor}");
voteGranted = false;
}
else if (this.journal.LastLogEntryId > request.LastLogEntryId)
{
this.logger.LogWarn(
$"Denying vote {request}: Local log is more up-to-date than candidate's log. "
+ $"{this.journal.LastLogEntryId} > {request.LastLogEntryId}");
voteGranted = false;
}
else
{
this.logger.LogInfo(
$"Granting vote to {request.Candidate} with last log: {request.LastLogEntryId}.");
this.messagesSinceLastElectionExpiry++;
voteGranted = true;
// If a vote has not yet been granted for this candidate in the current term,
// record that the vote is being granted and update the term.
if (this.persistentState.VotedFor != request.Candidate ||
this.persistentState.CurrentTerm != request.Term)
{
await this.persistentState.UpdateTermAndVote(request.Candidate, request.Term);
}
}
}
return(new RequestVoteResponse {
VoteGranted = voteGranted, Term = this.persistentState.CurrentTerm
});
}
public Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
return(Task.FromResult(new RequestVoteResponse()
{
CurrentTerm = request.CurrentTerm,
VoteGranted = false
}));
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
if (_linkedTokenSource.IsCancellationRequested)
{
return;
}
_sender.Send(dest, req);
}
public override Task <RequestVoteReply> RequestVote(RequestVoteRequest req, ServerCallContext _)
{
man.CheckFreeze();
var res = new RequestVoteReply {
Accept = man.RcvRequestVote(req.IdPart, req.Term, req.Tag)
};
Lib.Sleep(new Random().Next(minDelay, maxDelay));
return(Task.FromResult(res));
}
public async Task <RequestVoteResponse> RequestVote(RequestVoteRequest request)
{
// If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower (§5.1)
if (await this.coordinator.StepDownIfGreaterTerm(request))
{
return(await this.coordinator.Role.RequestVote(request));
}
return(new RequestVoteResponse {
VoteGranted = false, Term = this.persistentState.CurrentTerm
});
}
private async Task RequestVotes()
{
// Send RequestVote RPCs to all other servers.
var request = new RequestVoteRequest(
this.persistentState.CurrentTerm,
this.identity.Id,
this.journal.LastLogEntryId);
var tasks = new List <Task <RequestVoteResponse> >(this.membershipProvider.OtherServers.Count + 1)
{
this.cancellation.Token.WhenCanceled <RequestVoteResponse>()
};
// Send vote requests to each server.
foreach (var server in this.membershipProvider.OtherServers)
{
var serverGrain = this.grainFactory.GetGrain <IRaftGrain <TOperation> >(server);
tasks.Add(serverGrain.RequestVote(request));
}
// Wait for each server to respond.
while (!this.cancellation.IsCancellationRequested)
{
// Wait for one task to complete and remove it from the list.
var task = await Task.WhenAny(tasks);
tasks.Remove(task);
var response = await task;
if (await this.local.StepDownIfGreaterTerm(response))
{
return;
}
if (!response.VoteGranted)
{
continue;
}
this.votes++;
this.logger.LogInfo(
$"Received {this.votes} votes as candidate for term {this.persistentState.CurrentTerm}.");
// If votes received from majority of servers: become leader (§5.2)
if (this.votes >= this.QuorumSize)
{
this.logger.LogInfo(
$"Becoming leader for term {this.persistentState.CurrentTerm} with {this.votes}/{this.membershipProvider.OtherServers.Count + 1} votes.");
await this.local.BecomeLeader();
return;
}
}
}
public async Task LearnOfNewTermThroughRequestVote()
{
await this.role.Enter();
this.persistentState.CurrentTerm.Returns(_ => 1);
this.persistentState.VotedFor.Returns(_ => this.identity.Id);
var request = new RequestVoteRequest(2, "Napoleon", default(LogEntryId));
await this.role.RequestVote(request);
await this.coordinator.Received().StepDownIfGreaterTerm(request);
}
public CandidateBehavior()
{
PromoteToCandidate();
_candidateTerm = _state.CurrentTerm;
var voteRequest = new RequestVoteRequest(_state.Node, _candidateTerm, _state.LastCommitIndex, _state.LastCommitTerm);
foreach (var peer in _state.Peers)
{
peer.ControlEndpoint.Send(voteRequest);
}
// ScheduleElectionExpiration(_electionTimeout * 2);
}
public CandidateBehavior()
{
PromoteToCandidate();
_candidateTerm = _state.CurrentTerm;
var voteRequest = new RequestVoteRequest(_state.Node, _candidateTerm, _state.LastCommitIndex, _state.LastCommitTerm);
foreach (var peer in _state.Peers)
{
peer.ControlEndpoint.Send(voteRequest);
}
// ScheduleElectionExpiration(_electionTimeout * 2);
}
public async Task <RequestVoteResponse> RequestVote(RequestVoteRequest request)
{
// If the term of the requester is greater than the term of this instance, step down and handle the
// message as a follower.
if (await this.local.StepDownIfGreaterTerm(request))
{
return(await this.local.Role.RequestVote(request));
}
// Candidates vote for themselves and no other.
return(new RequestVoteResponse {
VoteGranted = false, Term = this.persistentState.CurrentTerm
});
}
public RequestVoteResponse RequestVote(RequestVoteRequest voteRequest)
{
if (voteRequest.Term <= _node.Data.CurrentTerm)
{
return(new RequestVoteResponse
{
Term = _node.Data.CurrentTerm,
VoteGranted = false
});
}
_nodePublisher.PublishEvent(new NodeCommandScheduled
{
Command = new SetNewTerm
{
Term = voteRequest.Term
}
}).Wait();
return(null);
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("request vote from " + dest, async() =>
{
var requestUri = string.Format("raft/requestVote?term={0}&lastLogIndex={1}&lastLogTerm={2}&trialOnly={3}&forcedElection={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LastLogIndex, req.LastLogTerm, req.TrialOnly, req.ForcedElection, req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.GetAsync(requestUri);
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error requesting vote from {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var requestVoteResponse = JsonConvert.DeserializeObject <RequestVoteResponse>(reply);
SendToSelf(requestVoteResponse);
});
}
}
public void RequestVoteAmendsTermOnRaftNodeWhenTermIsGreaterThanCurrentTerm()
{
// Arrange
var message = new RequestVoteRequest
{
Term = 1
};
var raftNode = Substitute.For <INode>();
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
raftNode.Data.Returns(new NodeData());
// Act
service.RequestVote(message);
// Assert
nodePublisher.Events.Should().HaveCount(1);
nodePublisher.Events[0].Command.Should().BeOfType <SetNewTerm>();
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
_parent.AddToQueue(this, dest.Name, req);
}
public RequestVoteResponse Handle(RequestVoteRequest req)
{
if (FromOurTopology(req) == false)
{
_log.Info("Got a request vote message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = "Cannot vote for a node outside my topology, my topology id is: " + Engine.CurrentTopology.TopologyId,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = false
});
}
//disregard RequestVoteRequest if this node receives regular messages and the leader is known
// Raft paper section 6 (cluster membership changes), this apply only if we are a follower, because
// candidate and leaders both generate their own heartbeat messages
var timeSinceLastHeartbeat = (DateTime.UtcNow - LastMessageTime).TotalMilliseconds;
var halfTimeout = (long)(Timeout / 2);
if (State == RaftEngineState.Follower && req.ForcedElection == false &&
(timeSinceLastHeartbeat < halfTimeout) && Engine.CurrentLeader != null)
{
_log.Info("Received RequestVoteRequest from a node within election timeout while leader exists, rejecting ");
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = "I currently have a leader and I am receiving heartbeats within election timeout.",
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = false
});
}
if (Engine.CurrentTopology.IsVoter(req.From) == false
// if it isn't in my cluster, but we don't have any voters, than we are probably a new node, so we'll accept this
// and allow it to become our leader
&& Engine.CurrentTopology.HasVoters)
{
_log.Info("Received RequestVoteRequest from a node that isn't a voting member in the cluster: {0}, rejecting", req.From);
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = "You are not a memeber in my cluster, and cannot be a leader",
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = false
});
}
_log.Debug("Received RequestVoteRequest, req.CandidateId = {0}, term = {1}", req.From, req.Term);
if (req.Term < Engine.PersistentState.CurrentTerm)
{
var msg = string.Format("Rejecting request vote because term {0} is lower than current term {1}",
req.Term, Engine.PersistentState.CurrentTerm);
_log.Info(msg);
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = msg,
From = Engine.Name,
TrialOnly = req.TrialOnly,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TermIncreaseMightGetMyVote = false
});
}
if (req.Term > Engine.PersistentState.CurrentTerm && req.TrialOnly == false)
{
Engine.UpdateCurrentTerm(req.Term, null);
}
if (Engine.PersistentState.VotedFor != null && Engine.PersistentState.VotedFor != req.From &&
Engine.PersistentState.VotedForTerm >= req.Term)
{
var msg = string.Format("Rejecting request vote because already voted for {0} in term {1}",
Engine.PersistentState.VotedFor, req.Term);
_log.Info(msg);
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = msg,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = true
});
}
if (Engine.LogIsUpToDate(req.LastLogTerm, req.LastLogIndex) == false)
{
var msg = string.Format("Rejecting request vote because remote log for {0} in not up to date.", req.From);
_log.Info(msg);
return(new RequestVoteResponse
{
VoteGranted = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = msg,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = false
});
}
if (req.TrialOnly == false)
{
// we said we would be voting for this guy, so we can give it a full election timeout,
// by treating this as a heart beat. This means we won't be timing out ourselves and trying
// to become the leader
LastHeartbeatTime = DateTime.UtcNow;
LastMessageTime = DateTime.UtcNow;
_log.Info("Recording vote for candidate = {0}", req.From);
Engine.PersistentState.RecordVoteFor(req.From, req.Term);
}
else
{
_log.Info("Voted for candidate = {0} in trial election for term {1}", req.From, req.Term);
}
return(new RequestVoteResponse
{
VoteGranted = true,
CurrentTerm = Engine.PersistentState.CurrentTerm,
VoteTerm = req.Term,
Message = "Vote granted",
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
TrialOnly = req.TrialOnly,
TermIncreaseMightGetMyVote = false
});
}
public Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
Thread.Sleep(10000);
throw new NotImplementedException();
}
public Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
throw new Exception("I am Angry!");
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
_sender.Send(dest, req);
}
/// <summary>
///
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <RequestVoteResponse> RequestVoteAsync(RequestVoteRequest request)
{
return(await _rpcClient.RequestVoteAsync(request));
}
public Task <RequestVoteResponse> RequestVote(RequestVoteRequest request) => this.coordinator.Role.RequestVote(request);
