public static AppendEntriesRequestWithEntries FromAppendEntriesRequest(AppendEntriesRequest message) { return(new AppendEntriesRequestWithEntries { Term = message.Term, PrevLogIndex = message.PrevLogIndex, PrevLogTerm = message.PrevLogTerm, Entries = message.Entries, LeaderCommit = message.LeaderCommit, From = message.From, ClusterTopologyId = message.ClusterTopologyId }); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("append entries to " + dest, async () => { var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}", req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId); var httpResponseMessage = await client.PostAsync(requestUri,new EntriesContent(req.Entries)); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var appendEntriesResponse = JsonConvert.DeserializeObject<AppendEntriesResponse>(reply); SendToSelf(appendEntriesResponse); }); } }
private void SendEntriesToPeer(NodeConnectionInfo peer) { LogEntry prevLogEntry; LogEntry[] entries; var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value if (Engine.StateMachine.SupportSnapshots && nextIndex != -1) { var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex(); if (snapshotIndex != null && nextIndex < snapshotIndex) { if (_snapshotsPendingInstallation.ContainsKey(peer.Name)) return; using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter()) { Engine.Transport.Send(peer, new CanInstallSnapshotRequest { From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, Index = snapshotWriter.Index, Term = snapshotWriter.Term, }); } return; } } if (nextIndex == -1) nextIndex = 0; try { entries = Engine.PersistentState.LogEntriesAfter(nextIndex) .Take(Engine.Options.MaxEntriesPerRequest) .ToArray(); prevLogEntry = entries.Length == 0 ? Engine.PersistentState.LastLogEntry() : Engine.PersistentState.GetLogEntry(entries[0].Index - 1); } catch (Exception e) { _log.Error("Error while fetching entries from persistent state.", e); throw; } if (_log.IsDebugEnabled) { _log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index); } var aer = new AppendEntriesRequest { Entries = entries, LeaderCommit = Engine.CommitIndex, PrevLogIndex = prevLogEntry.Index, PrevLogTerm = prevLogEntry.Term, Term = Engine.PersistentState.CurrentTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; Engine.Transport.Send(peer, aer); Engine.OnEntriesAppended(entries); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { _sender.Send(dest, req); }
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req) { if (FromOurTopology(req) == false) { _log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId); return new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, LeaderId = Engine.CurrentLeader, Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; } if (req.Term < Engine.PersistentState.CurrentTerm) { var msg = string.Format( "Rejecting append entries because msg term {0} is lower then current term: {1}", req.Term, Engine.PersistentState.CurrentTerm); _log.Info(msg); return new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, LeaderId = Engine.CurrentLeader, Message = msg, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; } if (req.Term > Engine.PersistentState.CurrentTerm) { Engine.UpdateCurrentTerm(req.Term, req.From); } if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false) { Engine.CurrentLeader = req.From; Engine.SetState(RaftEngineState.Follower); } var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0; if (prevTerm != req.PrevLogTerm) { var msg = string.Format( "Rejecting append entries because msg previous term {0} is not the same as the persisted current term {1} at log index {2}", req.PrevLogTerm, prevTerm, req.PrevLogIndex); _log.Info(msg); return new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, Message = msg, LeaderId = Engine.CurrentLeader, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; } LastHeartbeatTime = DateTime.UtcNow; LastMessageTime = DateTime.UtcNow; if (req.Entries.Length > 0) { _log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length, Engine.State); foreach (var logEntry in req.Entries) { _log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term); } // if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat // to process a message). In this case, our log already have the entries in question, and it would be a waste to // truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term // values in our log and in the entries, and then skip over the duplicates. var skip = 0; for (int i = 0; i < req.Entries.Length; i++) { var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1; if (termForEntry != req.Entries[i].Term) break; skip++; } if (skip != req.Entries.Length) Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip); var topologyChange = req.Entries.LastOrDefault(x => x.IsTopologyChange == true); // we consider the latest topology change to be in effect as soon as we see it, even before the // it is committed, see raft spec section 6: // a server always uses the latest configuration in its log, // regardless of whether the entry is committed if (topologyChange != null) { var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data); var topologyChangeCommand = command as TopologyChangeCommand; if (topologyChangeCommand == null) //precaution,should never be true //if this is true --> it is a serious issue and should be fixed immediately! throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand. Instead, it is of type: " + command.GetType() + ". It is probably a bug!"); _log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}", topologyChangeCommand.Requested); Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index); Engine.StartTopologyChange(topologyChangeCommand); } } var lastIndex = req.Entries.Length == 0 ? Engine.PersistentState.LastLogEntry().Index : req.Entries[req.Entries.Length - 1].Index; try { var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex); if (nextCommitIndex > Engine.CommitIndex) { CommitEntries(req.Entries, nextCommitIndex); } return new AppendEntriesResponse { Success = true, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; } catch (Exception e) { return new AppendEntriesResponse { Success = false, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = Engine.PersistentState.LastLogEntry().Index, Message = "Failed to apply new entries. Reason: " + e, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; } }
public override AppendEntriesResponse Handle(AppendEntriesRequest req) { if (_installingSnapshot == null) { return base.Handle(req); } var lastLogEntry = Engine.PersistentState.LastLogEntry(); return new AppendEntriesResponse { From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, CurrentTerm = Engine.PersistentState.CurrentTerm, LastLogIndex = lastLogEntry.Index, LeaderId = Engine.CurrentLeader, Message = "I am in the process of receiving a snapshot, so I cannot accept new entries at the moment", Success = false }; }