public static AppendEntriesRequestWithEntries FromAppendEntriesRequest(AppendEntriesRequest message)
{
return(new AppendEntriesRequestWithEntries
{
Term = message.Term,
PrevLogIndex = message.PrevLogIndex,
PrevLogTerm = message.PrevLogTerm,
Entries = message.Entries,
LeaderCommit = message.LeaderCommit,
From = message.From,
ClusterTopologyId = message.ClusterTopologyId
});
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("append entries to " + dest, async () =>
{
var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.PostAsync(requestUri,new EntriesContent(req.Entries));
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var appendEntriesResponse = JsonConvert.DeserializeObject<AppendEntriesResponse>(reply);
SendToSelf(appendEntriesResponse);
});
}
}
private void SendEntriesToPeer(NodeConnectionInfo peer)
{
LogEntry prevLogEntry;
LogEntry[] entries;
var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value
if (Engine.StateMachine.SupportSnapshots && nextIndex != -1)
{
var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex();
if (snapshotIndex != null && nextIndex < snapshotIndex)
{
if (_snapshotsPendingInstallation.ContainsKey(peer.Name))
return;
using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter())
{
Engine.Transport.Send(peer, new CanInstallSnapshotRequest
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
Index = snapshotWriter.Index,
Term = snapshotWriter.Term,
});
}
return;
}
}
if (nextIndex == -1)
nextIndex = 0;
try
{
entries = Engine.PersistentState.LogEntriesAfter(nextIndex)
.Take(Engine.Options.MaxEntriesPerRequest)
.ToArray();
prevLogEntry = entries.Length == 0
? Engine.PersistentState.LastLogEntry()
: Engine.PersistentState.GetLogEntry(entries[0].Index - 1);
}
catch (Exception e)
{
_log.Error("Error while fetching entries from persistent state.", e);
throw;
}
if (_log.IsDebugEnabled)
{
_log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index);
}
var aer = new AppendEntriesRequest
{
Entries = entries,
LeaderCommit = Engine.CommitIndex,
PrevLogIndex = prevLogEntry.Index,
PrevLogTerm = prevLogEntry.Term,
Term = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
Engine.Transport.Send(peer, aer);
Engine.OnEntriesAppended(entries);
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
_sender.Send(dest, req);
}
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req)
{
if (FromOurTopology(req) == false)
{
_log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.CurrentLeader,
Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
}
if (req.Term < Engine.PersistentState.CurrentTerm)
{
var msg = string.Format(
"Rejecting append entries because msg term {0} is lower then current term: {1}",
req.Term, Engine.PersistentState.CurrentTerm);
_log.Info(msg);
return new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.CurrentLeader,
Message = msg,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
}
if (req.Term > Engine.PersistentState.CurrentTerm)
{
Engine.UpdateCurrentTerm(req.Term, req.From);
}
if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false)
{
Engine.CurrentLeader = req.From;
Engine.SetState(RaftEngineState.Follower);
}
var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0;
if (prevTerm != req.PrevLogTerm)
{
var msg = string.Format(
"Rejecting append entries because msg previous term {0} is not the same as the persisted current term {1} at log index {2}",
req.PrevLogTerm, prevTerm, req.PrevLogIndex);
_log.Info(msg);
return new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
Message = msg,
LeaderId = Engine.CurrentLeader,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
}
LastHeartbeatTime = DateTime.UtcNow;
LastMessageTime = DateTime.UtcNow;
if (req.Entries.Length > 0)
{
_log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length,
Engine.State);
foreach (var logEntry in req.Entries)
{
_log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term);
}
// if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat
// to process a message). In this case, our log already have the entries in question, and it would be a waste to
// truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term
// values in our log and in the entries, and then skip over the duplicates.
var skip = 0;
for (int i = 0; i < req.Entries.Length; i++)
{
var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1;
if (termForEntry != req.Entries[i].Term)
break;
skip++;
}
if (skip != req.Entries.Length)
Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip);
var topologyChange = req.Entries.LastOrDefault(x => x.IsTopologyChange == true);
// we consider the latest topology change to be in effect as soon as we see it, even before the
// it is committed, see raft spec section 6:
// a server always uses the latest configuration in its log,
// regardless of whether the entry is committed
if (topologyChange != null)
{
var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data);
var topologyChangeCommand = command as TopologyChangeCommand;
if (topologyChangeCommand == null) //precaution,should never be true
//if this is true --> it is a serious issue and should be fixed immediately!
throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand.
Instead, it is of type: " + command.GetType() + ". It is probably a bug!");
_log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}",
topologyChangeCommand.Requested);
Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index);
Engine.StartTopologyChange(topologyChangeCommand);
}
}
var lastIndex = req.Entries.Length == 0 ?
Engine.PersistentState.LastLogEntry().Index :
req.Entries[req.Entries.Length - 1].Index;
try
{
var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex);
if (nextCommitIndex > Engine.CommitIndex)
{
CommitEntries(req.Entries, nextCommitIndex);
}
return new AppendEntriesResponse
{
Success = true,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
}
catch (Exception e)
{
return new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
Message = "Failed to apply new entries. Reason: " + e,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
}
}
public override AppendEntriesResponse Handle(AppendEntriesRequest req)
{
if (_installingSnapshot == null)
{
return base.Handle(req);
}
var lastLogEntry = Engine.PersistentState.LastLogEntry();
return new AppendEntriesResponse
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogEntry.Index,
LeaderId = Engine.CurrentLeader,
Message = "I am in the process of receiving a snapshot, so I cannot accept new entries at the moment",
Success = false
};
}
