public override RaftEventResult ReceiveAppendEntries(AppendEntriesRPC <T> appendEntries) { //Reply false if term from append entires < currentTerm (§5.1) if (appendEntries.LeaderTerm < CurrentTerm) { var falseResponse = new AppendEntriesResponse(CurrentTerm, Node.Id, false); return(RaftEventResult.ReplyMessage(falseResponse).SetTimer(Node.RaftSettings.FollowerTimeoutFrom, Node.RaftSettings.FollowerTimeoutTo)); } CurrentTerm = appendEntries.LeaderTerm; //if (appendEntries.LogEntries != null) // for (int i = 0; i < appendEntries.LogEntries.Count; i++) // { // if (appendEntries.LogEntries[i].CommitIndex > Node.Log.Count) // { // Node.Log.Add(appendEntries.LogEntries[i]); // Node.CurrentIndex = appendEntries.LogEntries[i].CommitIndex; // } // } var aeResponse = new AppendEntriesResponse(CurrentTerm, Node.Id, true); return(RaftEventResult.ReplyMessage(aeResponse).SetTimer(Node.RaftSettings.FollowerTimeoutFrom, Node.RaftSettings.FollowerTimeoutTo)); }
private void PromoteNodeToVoter(AppendEntriesResponse resp) { // if we got a successful append entries response from a promotable node, and it has caught up // with the committed entries, it means that we can promote it to voting positions, since it // can now become a leader. var upgradedNode = Engine.CurrentTopology.GetNodeByName(resp.From); if (upgradedNode == null) { return; } var requestTopology = new Topology( Engine.CurrentTopology.TopologyId, Engine.CurrentTopology.AllVotingNodes.Union(new[] { upgradedNode }), Engine.CurrentTopology.NonVotingNodes, Engine.CurrentTopology.PromotableNodes.Where(x => x != upgradedNode) ); if (Engine.CurrentlyChangingTopology() == false) { _log.Info( "Node {0} is a promotable node, and it has caught up to the current cluster commit index, but we are currently updating the topology, will try again later", resp.From); return; } _log.Info( "Node {0} is a promotable node, and it has caught up to the current cluster commit index, promoting to voting member", resp.From); Engine.ModifyTopology(requestTopology); }
public void server_should_become_follower_if_receives_greater_term_in_append_entries_response_message() { var appendEntriesResponse = new AppendEntriesResponse(20, false, Guid.NewGuid(), Guid.NewGuid()); var sendHeartbeat = new SendHeartbeat(); var remoteServers = new List <ServerInCluster> { new ServerInCluster(Guid.NewGuid()), new ServerInCluster(Guid.NewGuid()), new ServerInCluster(Guid.NewGuid()), new ServerInCluster(Guid.NewGuid()), new ServerInCluster(Guid.NewGuid()), }; var requestVoteResponses = remoteServers.Select(x => Task.FromResult(new RequestVoteResponse(0, true, x.Id, remoteServers[0].Id))).ToList(); this.Given(x => GivenTheFollowingRemoteServers(remoteServers)) .And(x => GivenANewServer()) .And(x => x.TheResponseIs(appendEntriesResponse)) .And(x => x.TheResponseIs(requestVoteResponses)) .And(x => ServerReceives(new BecomeCandidate(Guid.Empty))) .When(x => ServerReceives(sendHeartbeat)) .Then(x => ThenTheCurrentTermIs(20)) .And(x => TheServerIsAFollower()) .BDDfy(); }
public void Send(JsonOperationContext context, AppendEntriesResponse aer) { if (_log.IsInfoEnabled) { if (aer.Message != null) { _log.Info($"Replying with success {aer.Success}: {aer.Message}"); } else if (aer.Pending) { _log.Info($"Replying with pending for {aer.CurrentTerm} / {aer.LastLogIndex}"); } } var msg = new DynamicJsonValue { ["Type"] = nameof(AppendEntriesResponse), [nameof(AppendEntriesResponse.Success)] = aer.Success, [nameof(AppendEntriesResponse.Pending)] = aer.Pending, [nameof(AppendEntriesResponse.Message)] = aer.Message, [nameof(AppendEntriesResponse.CurrentTerm)] = aer.CurrentTerm, [nameof(AppendEntriesResponse.LastLogIndex)] = aer.LastLogIndex, }; Send(context, msg); }
private void WhenTheServerReceivesFailThenASuccessFromARemoteServer(ServerInCluster serverInCluster) { var fail = new AppendEntriesResponse(_server.CurrentTerm, false, serverInCluster.Id, _server.Id); var success = new AppendEntriesResponse(_server.CurrentTerm, true, serverInCluster.Id, _server.Id); _messageBus.Setup(x => x.Send(It.IsAny <AppendEntries>())).ReturnsInOrder(Task.FromResult(fail), Task.FromResult(success)); }
public override RaftEventResult ReceiveAppendEntriesResponse(AppendEntriesResponse appendEntriesResponse) { if (appendEntriesResponse.FollowerTerm > CurrentTerm) { return(Node.TranslateToState(RaftNodeState.Follower)); } return(RaftEventResult.Empty); }
public void SendToSelf(AppendEntriesResponse resp) { if (_linkedTokenSource.IsCancellationRequested) { return; } _bus.SendToSelf(resp); }
public void server_should_reply_true_if_entries_is_empty() { var id = Guid.NewGuid(); var appendEntries = new AppendEntries(0, id, 0, 0, null, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .When(x => ServerReceives(appendEntries)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheCurrentTermIs(0)) .BDDfy(); }
public void server_should_set_current_term_as_message_term_if_greater_than_current_term() { var id = Guid.NewGuid(); var appendEntries = new AppendEntries(1, id, 0, 0, null, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(1, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .When(x => ServerReceives(appendEntries)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheCurrentTermIs(1)) .BDDfy(); }
public void server_should_append_new_entries_not_in_log_and_reply_true() { var entries = new Log(0, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var appendEntries = new AppendEntries(0, id, 1, 0, entries, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .When(x => ServerReceives(appendEntries)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheLogContainsEntriesCount(1)) .BDDfy(); }
public void server_should_reply_false_if_term_is_less_than_current_term() { var entries = new Log(1, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var appendEntries = new AppendEntries(0, id, 0, 0, entries, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(1, false, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .And(x => GivenTheCurrentTermIs(1)) .When(x => ServerReceives(appendEntries)) .Then(x => ThenTheReplyIs(expected)) .BDDfy(); }
public void should_set_commit_index_if_leader_commit_greater_than_commit_index() { var entries = new Log(0, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var appendEntries = new AppendEntries(0, id, 1, 0, entries, 1, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .When(x => ServerReceives(appendEntries)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheCommitIndexIs(1)) .BDDfy(); }
public override void Handle(AppendEntriesResponse resp) { base.Handle(resp); var maxIndexOnQuorom = GetMaxIndexOnQuorum(); var lastLogEntry = Engine.PersistentState.LastLogEntry(); if (maxIndexOnQuorom >= lastLogEntry.Index) { _log.Info("Done sending all events to the cluster, can step down gracefully now"); TransferToBestMatch(); } }
private void UpdateNodeIndexes(AppendEntriesResponse resp, long defaultNextIndex, long defaultMatchIndex) { if (resp.MidpointIndex == null || resp.MidpointTerm == null) // no information, just go back one step { _nextIndexes[resp.From] = defaultNextIndex; _matchIndexes[resp.From] = defaultMatchIndex; if (_log.IsDebugEnabled) { _log.Debug($"UpdateNodeIndexes: No midpoint index. Using default next index {defaultNextIndex}"); } } else { var midpointIndex = resp.MidpointIndex.Value; var myMidpointTerm = Engine.PersistentState.TermFor(midpointIndex) ?? 0; var indexDiff = (resp.LastLogIndex - midpointIndex) / 2; indexDiff = indexDiff == 0 ? 1 : Math.Abs(indexDiff); if (myMidpointTerm == resp.MidpointTerm.Value) { // we know that we are a match on the middle, so let us set the // next attempt to be half way from the midpoint to the end _nextIndexes[resp.From] = midpointIndex + indexDiff; _matchIndexes[resp.From] = midpointIndex; if (_log.IsDebugEnabled) { _log.Debug($"UpdateNodeIndexes: Got match for mindpoint index: {midpointIndex}, term: {myMidpointTerm}."); } } else { // we don't have a match, so we need to go backward yet _nextIndexes[resp.From] = midpointIndex - indexDiff; _matchIndexes[resp.From] = 0; if (_log.IsDebugEnabled) { _log.Debug($"UpdateNodeIndexes: Got mismatch for mindpoint index: {midpointIndex}, leader term: {myMidpointTerm}, follower term: {resp.MidpointTerm.Value}"); } } } if (_log.IsDebugEnabled) { _log.Debug($"UpdateNodeIndexes operation result for {resp.From}: _nextIndexes = {_nextIndexes[resp.From]}, _matchIndexes = {_matchIndexes[resp.From]}."); } }
server_should_reply_false_if_log_doesnt_contain_an_entry_at_previous_log_index_matching_previous_log_term() { var entries = new Log(0, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var appendEntries = new AppendEntries(0, id, 0, 0, entries, 0, Guid.NewGuid()); var oldAppendEntries = new AppendEntries(0, id, 0, 1, entries, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, false, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .And(x => GivenTheServerRecieves(appendEntries)) .When(x => ServerReceives(oldAppendEntries)) .Then(x => ThenTheReplyIs(expected)) .BDDfy(); }
public void Send(JsonOperationContext context, AppendEntriesResponse aer) { if (_log.IsInfoEnabled) { _log.Info(aer.ToString()); } var msg = new DynamicJsonValue { ["Type"] = nameof(AppendEntriesResponse), [nameof(AppendEntriesResponse.Success)] = aer.Success, [nameof(AppendEntriesResponse.Pending)] = aer.Pending, [nameof(AppendEntriesResponse.Message)] = aer.Message, [nameof(AppendEntriesResponse.CurrentTerm)] = aer.CurrentTerm, [nameof(AppendEntriesResponse.LastLogIndex)] = aer.LastLogIndex, }; Send(context, msg); }
public void server_should_delete_existing_entry_and_all_that_follow_if_existing_entry_conflicts_with_a_new_one() { var initialEntries = new Log(0, new FakeCommand(Guid.NewGuid())); var initialAppendEntries = new AppendEntries(0, Guid.NewGuid(), 0, 0, initialEntries, 0, Guid.NewGuid()); var newEntries = new Log(1, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var newAppendEntries = new AppendEntries(0, id, 0, 0, newEntries, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .And(x => GivenTheServerRecieves(initialAppendEntries)) .When(x => ServerReceives(newAppendEntries)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheLogCountIs(1)) .BDDfy(); }
public void server_should_receive_multiple_append_entries() { var entry = new Log(0, new FakeCommand(Guid.NewGuid())); var id = Guid.NewGuid(); var first = new AppendEntries(0, id, 0, 0, entry, 0, Guid.NewGuid()); var second = new AppendEntries(0, id, 0, 0, entry, 0, Guid.NewGuid()); var third = new AppendEntries(0, id, 0, 0, entry, 0, Guid.NewGuid()); var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid()); this.Given(x => GivenANewServer()) .And(x => GivenTheServerRecieves(first)) .When(x => ServerReceives(second)) .And(x => ServerReceives(second)) .Then(x => ThenTheReplyIs(expected)) .And(x => ThenTheLogCountIs(3)) .BDDfy(); }
public AppendEntriesResponse Request(AppendEntries appendEntries) { AppendEntriesResponse response; if (AppendEntriesResponses.Count == 1) { response = new AppendEntriesResponse(_term, _appendEntryTwo); AppendEntriesResponses.Add(response); return(response); } if (AppendEntriesResponses.Count == 2) { response = new AppendEntriesResponse(_term, _appendEntryThree); AppendEntriesResponses.Add(response); return(response); } response = new AppendEntriesResponse(_term, _appendEntry); AppendEntriesResponses.Add(response); return(response); }
public ActionResult AppendEntries([FromBody] AppendEntriesEvent appendEntriesEvent) { IResponse response = null; if (_consensusContext.State == null) { response = new AppendEntriesResponse(_consensusContext.CurrentTerm, true); return(new ObjectResult(response)); } _consensusContext.State.TriggerEvent(appendEntriesEvent); if (appendEntriesEvent.Entries != null && appendEntriesEvent.Entries.Count > 0) { var isSuccessfull = _consensusContext.LogReplicable.OnLogReplication(appendEntriesEvent); response = new AppendEntriesResponse(_consensusContext.CurrentTerm, isSuccessfull); } else { response = new AppendEntriesResponse(_consensusContext.CurrentTerm, true); } return(new ObjectResult(response)); }
//peer级别 //[Authorize(Roles = "Peer")] public override async Task <NetResponse> AppendEntries(NetRequest request, ServerCallContext context) { try { var model = Newtonsoft.Json.JsonConvert.DeserializeObject <AppendEntries>(request.Data); var rs = await _raftNet.Handle(model); return(new NetResponse() { Data = Newtonsoft.Json.JsonConvert.SerializeObject(rs) }); } catch (Exception ex) { _logger.LogError(ex, ex.Message); var rs = new AppendEntriesResponse(); rs.Success = false; return(new NetResponse() { Data = Newtonsoft.Json.JsonConvert.SerializeObject(rs) }); } }
/// <summary> /// 接受来自leader节点的区块数据请求 /// </summary> /// <param name="appendEntries"></param> /// <returns></returns> public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries) { var response = new AppendEntriesResponse { Term = CurrentState.CurrentTerm, Success = false }; if (CurrentState.Id == appendEntries.LeaderId && CurrentState.CurrentTerm == appendEntries.Term ) { var log = _blockDataManager.GetBlockEntity(appendEntries.ChannelId, appendEntries.BlockHeight + 1); if (log != null) { response.Height = log.Header.Number; response.CurrentHash = log.Header.DataHash; response.BlockEntity = log; response.PreviousHash = log.Header.PreviousHash; response.Success = true; } } return(response); }
private void TheResponseIs(AppendEntriesResponse appendEntriesResponse) { _messageBus.Setup(x => x.Send(It.IsAny <AppendEntries>())).ReturnsAsync(appendEntriesResponse); }
public void SetAppendEntriesResponse(AppendEntriesResponse appendEntriesResponse) { _appendEntriesResponse = appendEntriesResponse; }
public virtual void Handle(AppendEntriesResponse resp) { // not a leader, no idea what to do with this. Probably an old // message from when we were a leader, ignoring. }
public void SendToSelf(AppendEntriesResponse resp) { _bus.SendToSelf(resp); }
public override RaftEventResult ReceiveAppendEntriesResponse(AppendEntriesResponse appendEntriesResponse) { return(RaftEventResult.Empty); }
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req) { var lastLogIndex = Engine.PersistentState.LastLogEntry().Index; if (FromOurTopology(req) == false) { _log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, LeaderId = Engine.CurrentLeader, Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term < Engine.PersistentState.CurrentTerm) { var msg = string.Format( "Rejecting append entries because msg term {0} is lower then current term: {1}", req.Term, Engine.PersistentState.CurrentTerm); _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, LeaderId = Engine.CurrentLeader, Message = msg, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term > Engine.PersistentState.CurrentTerm) { Engine.UpdateCurrentTerm(req.Term, req.From); } if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false) { Engine.CurrentLeader = req.From; Engine.SetState(RaftEngineState.Follower); } var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0; if (prevTerm != req.PrevLogTerm) { var midpointIndex = req.PrevLogIndex / 2; var midpointTerm = Engine.PersistentState.TermFor(midpointIndex) ?? 0; var msg = $"Rejecting append entries because msg previous term {req.PrevLogTerm} is not the same as the persisted current term {prevTerm}" + $" at log index {req.PrevLogIndex}. Midpoint index {midpointIndex}, midpoint term: {midpointTerm}"; _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = req.PrevLogIndex, Message = msg, LeaderId = Engine.CurrentLeader, MidpointIndex = midpointIndex, MidpointTerm = midpointTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } LastHeartbeatTime = DateTime.UtcNow; LastMessageTime = DateTime.UtcNow; var appendEntriesResponse = new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; if (req.Entries.Length > 0) { if (_log.IsDebugEnabled) { _log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length, Engine.State); foreach (var logEntry in req.Entries) { _log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term); } } // if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat // to process a message). In this case, our log already have the entries in question, and it would be a waste to // truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term // values in our log and in the entries, and then skip over the duplicates. var skip = 0; for (int i = 0; i < req.Entries.Length; i++) { var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1; if (termForEntry != req.Entries[i].Term) { break; } skip++; } var topologyChange = req.Entries.Skip(skip).LastOrDefault(x => x.IsTopologyChange == true); if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand != null && topologyChangeCommand.Requested.AllNodes.Select(x => x.Name).Contains(Engine.Options.SelfConnection.Name) == false) { _log.Warn("Got topology without self, disconnecting from the leader, clearing topology and moving to leader state"); var tcc = new TopologyChangeCommand { Requested = new Topology(Guid.NewGuid(), new[] { Engine.Options.SelfConnection }, new List <NodeConnectionInfo>(), new List <NodeConnectionInfo>()) }; Engine.PersistentState.SetCurrentTopology(tcc.Requested, 0L); Engine.StartTopologyChange(tcc); Engine.CommitTopologyChange(tcc); Engine.SetState(RaftEngineState.Leader); return(new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, Message = "Leaving cluster, because received topology from the leader that didn't contain us", From = Engine.Name, ClusterTopologyId = req.ClusterTopologyId, // we send this "older" ID, so the leader won't reject us }); } } if (skip != req.Entries.Length) { Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip); } else { // if we skipped the whole thing, this is fine, but let us hint to the leader that we are more // up to date then it thinks var lastReceivedIndex = req.Entries[req.Entries.Length - 1].Index; appendEntriesResponse.MidpointIndex = lastReceivedIndex + (lastLogIndex - lastReceivedIndex) / 2; appendEntriesResponse.MidpointTerm = Engine.PersistentState.TermFor(appendEntriesResponse.MidpointIndex.Value) ?? 0; _log.Info($"Got {req.Entries.Length} entires from index {req.Entries[0].Index} with term {req.Entries[0].Term} skipping all. " + $"Setting midpoint index to {appendEntriesResponse.MidpointIndex} with term {appendEntriesResponse.MidpointTerm}."); } // we consider the latest topology change to be in effect as soon as we see it, even before the // it is committed, see raft spec section 6: // a server always uses the latest con?guration in its log, // regardless of whether the entry is committed if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand == null) //precaution,should never be true //if this is true --> it is a serious issue and should be fixed immediately! { throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand. Instead, it is of type: " + command.GetType() + ". It is probably a bug!"); } _log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}", topologyChangeCommand.Requested); Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index); Engine.StartTopologyChange(topologyChangeCommand); } } var lastIndex = req.Entries.Length == 0 ? lastLogIndex : req.Entries[req.Entries.Length - 1].Index; try { var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex); if (nextCommitIndex > Engine.CommitIndex) { CommitEntries(req.Entries, nextCommitIndex); } appendEntriesResponse.LastLogIndex = lastLogIndex; return(appendEntriesResponse); } catch (Exception e) { return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, Message = "Failed to apply new entries. Reason: " + e, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } }
private void ThenTheReplyIs(AppendEntriesResponse expected) { _result.Success.ShouldBe(expected.Success); _result.Term.ShouldBe(expected.Term); }
private void ServerReceives(AppendEntries appendEntries) { _result = _server.Receive(appendEntries).Result; }