public async Task <HttpResponseMessage> AppendEntries([FromUri] AppendEntriesRequest request, [FromUri] int entriesCount) { var stream = await Request.Content.ReadAsStreamAsync(); request.Entries = new LogEntry[entriesCount]; for (int i = 0; i < entriesCount; i++) { var index = Read7BitEncodedInt(stream); var term = Read7BitEncodedInt(stream); var isTopologyChange = stream.ReadByte() == 1; var lengthOfData = (int)Read7BitEncodedInt(stream); request.Entries[i] = new LogEntry { Index = index, Term = term, IsTopologyChange = isTopologyChange, Data = new byte[lengthOfData] }; var start = 0; while (start < lengthOfData) { var read = stream.Read(request.Entries[i].Data, start, lengthOfData - start); start += read; } } var taskCompletionSource = new TaskCompletionSource <HttpResponseMessage>(); _bus.Publish(request, taskCompletionSource); return(await taskCompletionSource.Task); }
public void AppendEntriesAmendsTermOnRaftNodeWhenTermIsGreaterThanCurrentTerm() { // Arrange var message = new AppendEntriesRequest { Term = 1, PreviousLogIndex = 1, PreviousLogTerm = 0 }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Follower); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); var nodeData = new NodeData { Log = new NodeLog() }; nodeData.Log.SetLogEntry(1, 0); raftNode.Data.Returns(nodeData); // Act service.AppendEntries(message); // Assert nodePublisher.Events.Should().HaveCount(2); nodePublisher.Events[0].Command.Should().BeOfType <SetNewTerm>(); }
public void AppendEntriesReturnsFalseIfPrevEntryTermDoesNotMatch() { // Arrange var message = new AppendEntriesRequest { PreviousLogIndex = 0, PreviousLogTerm = 1 }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Follower); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); var raftLog = new NodeLog(); raftLog.SetLogEntry(1, 2L); raftNode.Data.Returns(new NodeData { Log = raftLog }); // Act var response = service.AppendEntries(message); // Assert response.Success.Should().BeFalse( "because node cannot apply log if the term " + "for the leaders previous log index does not match."); }
public void SetLeaderIdWhenFollowerAndAppendEntriesRecieved() { // Arrange var message = new AppendEntriesRequest { Term = 0, PreviousLogIndex = 0, PreviousLogTerm = 0 }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Follower); raftNode.Data.Returns(new NodeData()); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act service.AppendEntries(message); // Assert nodePublisher.Events.Should().HaveCount(1); nodePublisher.Events[0].Command.Should().BeOfType <SetLeaderInformation>(); }
public void CancelElectionIfCandidateAndReceiveAppendEntries() { // Arrange var message = new AppendEntriesRequest { Term = 0, PreviousLogIndex = 0, PreviousLogTerm = 0 }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Candidate); raftNode.Data.Returns(new NodeData()); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>(); nodePublisher.OnPublish(() => raftNode.CurrentState.Returns(NodeState.Follower)); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act service.AppendEntries(message); // Assert nodePublisher.Events.Should().HaveCount(2); nodePublisher.Events[0].Command.Should().BeOfType <CancelElection>(); }
public void AppendEntriesReturnsNodesCurrentTerm() { // Arrange const int expectedTerm = 456; var message = new AppendEntriesRequest(); var raftNode = Substitute.For <INode>(); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); raftNode.Data.Returns(new NodeData { Log = new NodeLog(), CurrentTerm = expectedTerm }); // Act var response = service.AppendEntries(message); // Assert response.Term.ShouldBeEquivalentTo(expectedTerm); }
public void ThrowsWhenAppendEntriesReceivedAndNodeWillNotBeTransitionedToFollower() { // Arrange var message = new AppendEntriesRequest { Term = 24, PreviousLogIndex = 0, PreviousLogTerm = 0 }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Leader); raftNode.Data.Returns(new NodeData { CurrentTerm = 24 }); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act var actAction = new Action(() => service.AppendEntries(message)); // Assert actAction.ShouldThrow <FaultException <MultipleLeadersForTermFault> >(); }
public override Task <AppendEntriesReply> AppendEntries(AppendEntriesRequest req, ServerCallContext _) { man.CheckFreeze(); var sInfo = new ServerInfo(req.Mid, req.Tag, req.Term); var entries = new List <KeyValuePair <string, string> >(); foreach (var e in req.Entries) { entries.Add(new KeyValuePair <string, string>(e.IdObj, e.Val)); } var mySInfo = man.RcvAppendEntries(req.IdPart, sInfo, entries); if (mySInfo.Term <= sInfo.Term) { foreach (var e in entries) { store.Write(req.IdPart, e.Key, e.Value); } } var res = new AppendEntriesReply { Term = mySInfo.Term, Tag = mySInfo.Tag, Mid = mySInfo.Mid }; Lib.Sleep(new Random().Next(minDelay, maxDelay)); return(Task.FromResult(res)); }
public ServerInfo AppendEntries(ServerInfo sInfo, string idPart, List <KeyValuePair <string, string> > entries) { var req = new AppendEntriesRequest() { Term = sInfo.Term, Tag = sInfo.Tag, Mid = sInfo.Mid, IdPart = idPart }; if (entries != null) { foreach (var e in entries) { req.Entries.Add(new Entry() { IdObj = e.Key, Val = e.Value }); } } try { var res = stub.AppendEntries(req); return(new ServerInfo(res.Mid, res.Tag, res.Term)); } catch (RpcException) { NegAvail(); return(null); } }
public void AppendEntriesIsUnsuccessfulIfLeaderTermIsLessThanCurrentTerm() { // Arrange var message = new AppendEntriesRequest { Term = 234 }; var nodeData = new NodeData { CurrentTerm = message.Term + 10, Log = new NodeLog() }; var raftNode = Substitute.For <INode>(); raftNode.Data.Returns(nodeData); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act var response = service.AppendEntries(message); // Assert response.Success.Should().BeFalse( "because node cannot apply log " + "if the leader term is less than the nodes term"); }
public void LogReplicatorCallsAppendEntriesPassingEncodedLog() { // Arrange var encodedLog = BitConverter.GetBytes(100); AppendEntriesRequest request = null; var service = Substitute.For <IRaftService>(); service.When(x => x.AppendEntries(Arg.Any <AppendEntriesRequest>())) .Do(x => request = x.Arg <AppendEntriesRequest>()); var peers = new List <PeerNode> { new PeerNode() }; var handler = new LogReplicator(peers); var @event = TestEventFactory.GetCommandEvent(1L, encodedLog); // Act handler.Handle(@event); // Assert request.Should().NotBeNull(); request.Entries.Should().HaveCount(1); request.Entries[0].Should().Equal(encodedLog); }
/// <summary> /// /// </summary> /// <param name="request"></param> /// <returns></returns> public async Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request) { try { var json = JsonConvert.SerializeObject(request); var content = new StringContent(json); content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); var response = await _httpClient.PostAsync("/_raft/appendEntries", content); if (response != null && response.IsSuccessStatusCode) { var responseContent = await response.Content.ReadAsStringAsync(); return(JsonConvert.DeserializeObject <AppendEntriesResponse>(responseContent)); } return(new AppendEntriesResponse(request.Term, false)); } catch (HttpRequestException requestEx) { _logger.LogError($"AppendEntries fail: {requestEx.Message}, serverUri: {_httpClient.BaseAddress}"); return(new AppendEntriesResponse(request.Term, false)); } catch (Exception ex) { _logger.LogError(ex, $"AppendEntries fail: {ex.Message}, serverUri: {_httpClient.BaseAddress}"); return(new AppendEntriesResponse(request.Term, false)); } }
private bool GrantSuccess(AppendEntriesRequest requestVote) { try { if (requestVote.Term < Peer.MyState.CurrentTerm) { return(false); } if (!EntryTermMatches(requestVote)) { return(false); } DeleteConflictEntries(requestVote.PrevLogIndex); if (requestVote.entries != null) { Peer.MyState.Log.AddRange(requestVote.entries); } if (requestVote.LeaderCommitIndex > Peer.MyState.CommittedIndex) { Peer.MyState.CommittedIndex = Math.Min(requestVote.LeaderCommitIndex, Peer.MyState.Log.Count - 1); WriteLine(Peer.MyState.CommittedIndex); } } catch (Exception exception) { WriteLine(exception.Message); } return(true); }
public AppendEntriesResponse AppendEntry(AppendEntriesRequest requestVote) { var electTime = ElectionTimeout; Election.Change(electTime, Timeout.Infinite); lock (_lockObject) { Peer.MyState.ThisServerInfo.PersistedState.SavePeerState(Peer.MyState); foreach (var peer in Peer.MyState.PeerlList) { peer.Value.Persist(Peer.MyState.ThisServerInfo); } var differentStateOrLeader = Peer.MyState.CurrentState != CurrentServerState.Follower || Peer.MyState.Leader != null && Peer.MyState.Leader.Id != requestVote.Leader.Id; if (requestVote.Term > Peer.MyState.CurrentTerm || Peer.MyState.CurrentTerm <= requestVote.Term && differentStateOrLeader) { WriteLine($"Server {Peer.MyState.Id} is reverting to follower from {Peer.MyState.CurrentState}"); Peer.MyState.CurrentTerm = requestVote.Term; Peer.MyState.CurrentState = CurrentServerState.Follower; Peer.MyState.Leader = requestVote.Leader; } var success = GrantSuccess(requestVote); return(new AppendEntriesResponse(Peer.MyState.CurrentTerm, success, Peer.MyState.ThisServerInfo.Id)); } }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { if (MaybeIgnoreFrequentRequestsIfServerDown(dest)) { return; } LogStatus("append entries to " + dest, async() => { var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}", req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId); using (var request = CreateRequest(dest, requestUri, HttpMethods.Post)) { var httpResponseMessage = await request.WriteAsync(() => new EntriesContent(req.Entries)).ConfigureAwait(false); UpdateConnectionFailureCounts(dest, httpResponseMessage); var reply = await httpResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var appendEntriesResponse = JsonConvert.DeserializeObject <AppendEntriesResponse>(reply); SendToSelf(appendEntriesResponse); } }); }
public override async Task<AppendEntriesResult> AppendEntries(AppendEntriesRequest request) { if (SleepOnAppendEntries) { await Task.Delay(Timeout.Infinite); } return await base.AppendEntries(request); }
public override async Task <AppendEntriesResult> AppendEntries(AppendEntriesRequest request) { if (SleepOnAppendEntries) { await Task.Delay(Timeout.Infinite); } return(await base.AppendEntries(request)); }
/// <summary> /// Processes the given append entries request. /// </summary> /// <param name="request">AppendEntriesRequest</param> void AppendEntries(AppendEntriesRequest request) { if (request.Term < this.CurrentTerm) { this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false, this.Id, request.ReceiverEndpoint)); } else { if (request.PrevLogIndex > 0 && (this.Logs.Count < request.PrevLogIndex || this.Logs[request.PrevLogIndex - 1].Term != request.PrevLogTerm)) { this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false, this.Id, request.ReceiverEndpoint)); } else { if (request.Entries.Count > 0) { var currentIndex = request.PrevLogIndex + 1; foreach (var entry in request.Entries) { if (this.Logs.Count < currentIndex) { this.Logs.Add(entry); } else if (this.Logs[currentIndex - 1].Term != entry.Term) { this.Logs.RemoveRange(currentIndex - 1, this.Logs.Count - (currentIndex - 1)); this.Logs.Add(entry); } currentIndex++; } } if (request.LeaderCommit > this.CommitIndex && this.Logs.Count < request.LeaderCommit) { this.CommitIndex = this.Logs.Count; } else if (request.LeaderCommit > this.CommitIndex) { this.CommitIndex = request.LeaderCommit; } if (this.CommitIndex > this.LastApplied) { this.LastApplied++; } this.LeaderId = request.LeaderId; this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, true, this.Id, request.ReceiverEndpoint)); } } }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { if (_linkedTokenSource.IsCancellationRequested) { return; } _sender.Send(dest, req); }
public override void Handle(CommandScheduled @event) { var request = new AppendEntriesRequest { Entries = new[] { @event.EncodedEntry } }; foreach (var peerNode in _peers) { GetChannel(peerNode).AppendEntries(request); } }
private bool EntryTermMatches(AppendEntriesRequest requestVote) { if (requestVote.PrevLogIndex < 0 && Peer.MyState.Log.Count == 0) { return(requestVote.PrevLogIndex == -1); } if (requestVote.PrevLogIndex >= Peer.MyState.Log.Count) { return(false); } return(requestVote.PrevLogIndex == -1 || Peer.MyState.Log[requestVote.PrevLogIndex].Term == requestVote.PrevLogTerm); }
public void PublishesToBufferAfterSafetyChecks() { // Arrange var message = new AppendEntriesRequest { Term = 11, PreviousLogIndex = 2, PreviousLogTerm = 13, LeaderCommit = 12, Entries = new [] { BitConverter.GetBytes(100), BitConverter.GetBytes(200) } }; var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Follower); var nodeData = new NodeData { Log = new NodeLog() }; nodeData.Log.SetLogEntry(message.PreviousLogIndex, message.PreviousLogTerm); raftNode.Data.Returns(nodeData); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act service.AppendEntries(message); // Assert appendEntriesPublisher.Received() .PublishEvent(Arg.Is <AppendEntriesRequested>(requested => requested.PreviousLogIndex == message.PreviousLogIndex && requested.PreviousLogTerm == message.PreviousLogTerm && requested.LeaderCommit == message.LeaderCommit && requested.Entries == message.Entries)); }
public override AppendEntriesResponse Handle(AppendEntriesRequest req) { if (_installingSnapshot == null) { return(base.Handle(req)); } var lastLogEntry = Engine.PersistentState.LastLogEntry(); return(new AppendEntriesResponse { From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogEntry.Index, LeaderId = Engine.CurrentLeader, Message = "I am in the process of receiving a snapshot, so I cannot accept new entries at the moment", Success = false }); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("append entries to " + dest, async() => { var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}", req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId); var httpResponseMessage = await client.PostAsync(requestUri, new EntriesContent(req.Entries)); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var appendEntriesResponse = JsonConvert.DeserializeObject <AppendEntriesResponse>(reply); SendToSelf(appendEntriesResponse); }); } }
public void AppendEntriesResetsTimer() { // Arrange var message = new AppendEntriesRequest(); var raftNode = Substitute.For <INode>(); raftNode.CurrentState.Returns(NodeState.Follower); raftNode.Data.Returns(new NodeData { Log = new NodeLog() }); var timer = Substitute.For <INodeTimer>(); var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >(); var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >(); var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode); // Act service.AppendEntries(message); // Assert timer.Received(1).ResetTimer(); }
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req) { if (FromOurTopology(req) == false) { _log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, LeaderId = Engine.CurrentLeader, Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term < Engine.PersistentState.CurrentTerm) { var msg = string.Format( "Rejecting append entries because msg term {0} is lower then current term: {1}", req.Term, Engine.PersistentState.CurrentTerm); _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, LeaderId = Engine.CurrentLeader, Message = msg, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term > Engine.PersistentState.CurrentTerm) { Engine.UpdateCurrentTerm(req.Term, req.From); } if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false) { Engine.CurrentLeader = req.From; Engine.SetState(RaftEngineState.Follower); } var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0; if (prevTerm != req.PrevLogTerm) { var msg = string.Format( "Rejecting append entries because msg previous term {0} is not the same as the persisted current term {1} at log index {2}", req.PrevLogTerm, prevTerm, req.PrevLogIndex); _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, Message = msg, LeaderId = Engine.CurrentLeader, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } LastHeartbeatTime = DateTime.UtcNow; LastMessageTime = DateTime.UtcNow; if (req.Entries.Length > 0) { _log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length, Engine.State); foreach (var logEntry in req.Entries) { _log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term); } // if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat // to process a message). In this case, our log already have the entries in question, and it would be a waste to // truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term // values in our log and in the entries, and then skip over the duplicates. var skip = 0; for (int i = 0; i < req.Entries.Length; i++) { var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1; if (termForEntry != req.Entries[i].Term) { break; } skip++; } if (skip != req.Entries.Length) { Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip); } var topologyChange = req.Entries.LastOrDefault(x => x.IsTopologyChange == true); // we consider the latest topology change to be in effect as soon as we see it, even before the // it is committed, see raft spec section 6: // a server always uses the latest configuration in its log, // regardless of whether the entry is committed if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand == null) //precaution,should never be true //if this is true --> it is a serious issue and should be fixed immediately! { throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand. Instead, it is of type: " + command.GetType() + ". It is probably a bug!"); } _log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}", topologyChangeCommand.Requested); Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index); Engine.StartTopologyChange(topologyChangeCommand); } } var lastIndex = req.Entries.Length == 0 ? Engine.PersistentState.LastLogEntry().Index : req.Entries[req.Entries.Length - 1].Index; try { var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex); if (nextCommitIndex > Engine.CommitIndex) { CommitEntries(req.Entries, nextCommitIndex); } return(new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } catch (Exception e) { return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, Message = "Failed to apply new entries. Reason: " + e, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } }
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req) { var lastLogIndex = Engine.PersistentState.LastLogEntry().Index; if (FromOurTopology(req) == false) { _log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, LeaderId = Engine.CurrentLeader, Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term < Engine.PersistentState.CurrentTerm) { var msg = string.Format( "Rejecting append entries because msg term {0} is lower then current term: {1}", req.Term, Engine.PersistentState.CurrentTerm); _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, LeaderId = Engine.CurrentLeader, Message = msg, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } if (req.Term > Engine.PersistentState.CurrentTerm) { Engine.UpdateCurrentTerm(req.Term, req.From); } if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false) { Engine.CurrentLeader = req.From; Engine.SetState(RaftEngineState.Follower); } var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0; if (prevTerm != req.PrevLogTerm) { var midpointIndex = req.PrevLogIndex / 2; var midpointTerm = Engine.PersistentState.TermFor(midpointIndex) ?? 0; var msg = $"Rejecting append entries because msg previous term {req.PrevLogTerm} is not the same as the persisted current term {prevTerm}" + $" at log index {req.PrevLogIndex}. Midpoint index {midpointIndex}, midpoint term: {midpointTerm}"; _log.Info(msg); return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = req.PrevLogIndex, Message = msg, LeaderId = Engine.CurrentLeader, MidpointIndex = midpointIndex, MidpointTerm = midpointTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } LastHeartbeatTime = DateTime.UtcNow; LastMessageTime = DateTime.UtcNow; var appendEntriesResponse = new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; if (req.Entries.Length > 0) { if (_log.IsDebugEnabled) { _log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length, Engine.State); foreach (var logEntry in req.Entries) { _log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term); } } // if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat // to process a message). In this case, our log already have the entries in question, and it would be a waste to // truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term // values in our log and in the entries, and then skip over the duplicates. var skip = 0; for (int i = 0; i < req.Entries.Length; i++) { var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1; if (termForEntry != req.Entries[i].Term) { break; } skip++; } var topologyChange = req.Entries.Skip(skip).LastOrDefault(x => x.IsTopologyChange == true); if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand != null && topologyChangeCommand.Requested.AllNodes.Select(x => x.Name).Contains(Engine.Options.SelfConnection.Name) == false) { _log.Warn("Got topology without self, disconnecting from the leader, clearing topology and moving to leader state"); var tcc = new TopologyChangeCommand { Requested = new Topology(Guid.NewGuid(), new[] { Engine.Options.SelfConnection }, new List <NodeConnectionInfo>(), new List <NodeConnectionInfo>()) }; Engine.PersistentState.SetCurrentTopology(tcc.Requested, 0L); Engine.StartTopologyChange(tcc); Engine.CommitTopologyChange(tcc); Engine.SetState(RaftEngineState.Leader); return(new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, Message = "Leaving cluster, because received topology from the leader that didn't contain us", From = Engine.Name, ClusterTopologyId = req.ClusterTopologyId, // we send this "older" ID, so the leader won't reject us }); } } if (skip != req.Entries.Length) { Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip); } else { // if we skipped the whole thing, this is fine, but let us hint to the leader that we are more // up to date then it thinks var lastReceivedIndex = req.Entries[req.Entries.Length - 1].Index; appendEntriesResponse.MidpointIndex = lastReceivedIndex + (lastLogIndex - lastReceivedIndex) / 2; appendEntriesResponse.MidpointTerm = Engine.PersistentState.TermFor(appendEntriesResponse.MidpointIndex.Value) ?? 0; _log.Info($"Got {req.Entries.Length} entires from index {req.Entries[0].Index} with term {req.Entries[0].Term} skipping all. " + $"Setting midpoint index to {appendEntriesResponse.MidpointIndex} with term {appendEntriesResponse.MidpointTerm}."); } // we consider the latest topology change to be in effect as soon as we see it, even before the // it is committed, see raft spec section 6: // a server always uses the latest con?guration in its log, // regardless of whether the entry is committed if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand == null) //precaution,should never be true //if this is true --> it is a serious issue and should be fixed immediately! { throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand. Instead, it is of type: " + command.GetType() + ". It is probably a bug!"); } _log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}", topologyChangeCommand.Requested); Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index); Engine.StartTopologyChange(topologyChangeCommand); } } var lastIndex = req.Entries.Length == 0 ? lastLogIndex : req.Entries[req.Entries.Length - 1].Index; try { var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex); if (nextCommitIndex > Engine.CommitIndex) { CommitEntries(req.Entries, nextCommitIndex); } appendEntriesResponse.LastLogIndex = lastLogIndex; return(appendEntriesResponse); } catch (Exception e) { return(new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogIndex, Message = "Failed to apply new entries. Reason: " + e, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }); } }
/// <summary> /// Processes the given append entries request. /// </summary> /// <param name="request">AppendEntriesRequest</param> void AppendEntries(AppendEntriesRequest request) { if (request.Term < this.CurrentTerm) { Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " + this.Logs.Count + " | last applied: " + this.LastApplied + " | append false (< term)\n"); this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false, this.Id, request.ReceiverEndpoint)); } else { if (request.PrevLogIndex > 0 && (this.Logs.Count < request.PrevLogIndex || this.Logs[request.PrevLogIndex - 1].Term != request.PrevLogTerm)) { Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " + this.Logs.Count + " | last applied: " + this.LastApplied + " | append false (not in log)\n"); this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false, this.Id, request.ReceiverEndpoint)); } else { if (request.Entries.Count > 0) { var currentIndex = request.PrevLogIndex + 1; foreach (var entry in request.Entries) { if (this.Logs.Count < currentIndex) { this.Logs.Add(entry); } else if (this.Logs[currentIndex - 1].Term != entry.Term) { this.Logs.RemoveRange(currentIndex - 1, this.Logs.Count - (currentIndex - 1)); this.Logs.Add(entry); } currentIndex++; } } if (request.LeaderCommit > this.CommitIndex && this.Logs.Count < request.LeaderCommit) { this.CommitIndex = this.Logs.Count; } else if (request.LeaderCommit > this.CommitIndex) { this.CommitIndex = request.LeaderCommit; } if (this.CommitIndex > this.LastApplied) { this.LastApplied++; } Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " + this.Logs.Count + " | entries received " + request.Entries.Count + " | last applied " + this.LastApplied + " | append true\n"); this.LeaderId = request.LeaderId; this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, true, this.Id, request.ReceiverEndpoint)); } } }
public async Task <AppendEntriesResult> AppendEntries([FromBody] AppendEntriesRequest request) { return(await node.AppendEntries(request)); }
/// <summary> /// /// </summary> /// <param name="request"></param> /// <returns></returns> public async Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request) { return(await _rpcClient.AppendEntriesAsync(request)); }
private void SendEntriesToPeer(NodeConnectionInfo peer) { LogEntry prevLogEntry; LogEntry[] entries; var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value if (Engine.StateMachine.SupportSnapshots && nextIndex != -1) { var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex(); if (snapshotIndex != null && nextIndex < snapshotIndex) { if (_snapshotsPendingInstallation.ContainsKey(peer.Name)) { return; } var snapshotWriter = Engine.StateMachine.GetSnapshotWriter(); Engine.Transport.Send(peer, new CanInstallSnapshotRequest { From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, Index = snapshotWriter.Index, Term = snapshotWriter.Term, }); return; } } if (nextIndex == -1) { nextIndex = 0; } try { entries = Engine.PersistentState.LogEntriesAfter(nextIndex) .Take(Engine.Options.MaxEntriesPerRequest) .ToArray(); prevLogEntry = entries.Length == 0 ? Engine.PersistentState.LastLogEntry() : Engine.PersistentState.GetLogEntry(entries[0].Index - 1); } catch (Exception e) { _log.Error("Error while fetching entries from persistent state.", e); throw; } if (_log.IsDebugEnabled) { _log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index); } var aer = new AppendEntriesRequest { Entries = entries, LeaderCommit = Engine.CommitIndex, PrevLogIndex = prevLogEntry.Index, PrevLogTerm = prevLogEntry.Term, Term = Engine.PersistentState.CurrentTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; Engine.Transport.Send(peer, aer); Engine.OnEntriesAppended(entries); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { _sender.Send(dest, req); }