public async Task <HttpResponseMessage> AppendEntries([FromUri] AppendEntriesRequest request, [FromUri] int entriesCount)
{
var stream = await Request.Content.ReadAsStreamAsync();
request.Entries = new LogEntry[entriesCount];
for (int i = 0; i < entriesCount; i++)
{
var index = Read7BitEncodedInt(stream);
var term = Read7BitEncodedInt(stream);
var isTopologyChange = stream.ReadByte() == 1;
var lengthOfData = (int)Read7BitEncodedInt(stream);
request.Entries[i] = new LogEntry
{
Index = index,
Term = term,
IsTopologyChange = isTopologyChange,
Data = new byte[lengthOfData]
};
var start = 0;
while (start < lengthOfData)
{
var read = stream.Read(request.Entries[i].Data, start, lengthOfData - start);
start += read;
}
}
var taskCompletionSource = new TaskCompletionSource <HttpResponseMessage>();
_bus.Publish(request, taskCompletionSource);
return(await taskCompletionSource.Task);
}
public void AppendEntriesAmendsTermOnRaftNodeWhenTermIsGreaterThanCurrentTerm()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 1,
PreviousLogIndex = 1,
PreviousLogTerm = 0
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Follower);
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
var nodeData = new NodeData
{
Log = new NodeLog()
};
nodeData.Log.SetLogEntry(1, 0);
raftNode.Data.Returns(nodeData);
// Act
service.AppendEntries(message);
// Assert
nodePublisher.Events.Should().HaveCount(2);
nodePublisher.Events[0].Command.Should().BeOfType <SetNewTerm>();
}
public void AppendEntriesReturnsFalseIfPrevEntryTermDoesNotMatch()
{
// Arrange
var message = new AppendEntriesRequest
{
PreviousLogIndex = 0,
PreviousLogTerm = 1
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Follower);
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
var raftLog = new NodeLog();
raftLog.SetLogEntry(1, 2L);
raftNode.Data.Returns(new NodeData {
Log = raftLog
});
// Act
var response = service.AppendEntries(message);
// Assert
response.Success.Should().BeFalse(
"because node cannot apply log if the term " +
"for the leaders previous log index does not match.");
}
public void SetLeaderIdWhenFollowerAndAppendEntriesRecieved()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 0,
PreviousLogIndex = 0,
PreviousLogTerm = 0
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Follower);
raftNode.Data.Returns(new NodeData());
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
service.AppendEntries(message);
// Assert
nodePublisher.Events.Should().HaveCount(1);
nodePublisher.Events[0].Command.Should().BeOfType <SetLeaderInformation>();
}
public void CancelElectionIfCandidateAndReceiveAppendEntries()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 0,
PreviousLogIndex = 0,
PreviousLogTerm = 0
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Candidate);
raftNode.Data.Returns(new NodeData());
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
nodePublisher.OnPublish(() => raftNode.CurrentState.Returns(NodeState.Follower));
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
service.AppendEntries(message);
// Assert
nodePublisher.Events.Should().HaveCount(2);
nodePublisher.Events[0].Command.Should().BeOfType <CancelElection>();
}
public void AppendEntriesReturnsNodesCurrentTerm()
{
// Arrange
const int expectedTerm = 456;
var message = new AppendEntriesRequest();
var raftNode = Substitute.For <INode>();
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
raftNode.Data.Returns(new NodeData
{
Log = new NodeLog(),
CurrentTerm = expectedTerm
});
// Act
var response = service.AppendEntries(message);
// Assert
response.Term.ShouldBeEquivalentTo(expectedTerm);
}
public void ThrowsWhenAppendEntriesReceivedAndNodeWillNotBeTransitionedToFollower()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 24,
PreviousLogIndex = 0,
PreviousLogTerm = 0
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Leader);
raftNode.Data.Returns(new NodeData {
CurrentTerm = 24
});
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
var actAction = new Action(() => service.AppendEntries(message));
// Assert
actAction.ShouldThrow <FaultException <MultipleLeadersForTermFault> >();
}
public override Task <AppendEntriesReply> AppendEntries(AppendEntriesRequest req, ServerCallContext _)
{
man.CheckFreeze();
var sInfo = new ServerInfo(req.Mid, req.Tag, req.Term);
var entries = new List <KeyValuePair <string, string> >();
foreach (var e in req.Entries)
{
entries.Add(new KeyValuePair <string, string>(e.IdObj, e.Val));
}
var mySInfo = man.RcvAppendEntries(req.IdPart, sInfo, entries);
if (mySInfo.Term <= sInfo.Term)
{
foreach (var e in entries)
{
store.Write(req.IdPart, e.Key, e.Value);
}
}
var res = new AppendEntriesReply {
Term = mySInfo.Term,
Tag = mySInfo.Tag,
Mid = mySInfo.Mid
};
Lib.Sleep(new Random().Next(minDelay, maxDelay));
return(Task.FromResult(res));
}
public ServerInfo AppendEntries(ServerInfo sInfo, string idPart, List <KeyValuePair <string, string> > entries)
{
var req = new AppendEntriesRequest()
{
Term = sInfo.Term,
Tag = sInfo.Tag,
Mid = sInfo.Mid,
IdPart = idPart
};
if (entries != null)
{
foreach (var e in entries)
{
req.Entries.Add(new Entry()
{
IdObj = e.Key, Val = e.Value
});
}
}
try {
var res = stub.AppendEntries(req);
return(new ServerInfo(res.Mid, res.Tag, res.Term));
} catch (RpcException) {
NegAvail();
return(null);
}
}
public void AppendEntriesIsUnsuccessfulIfLeaderTermIsLessThanCurrentTerm()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 234
};
var nodeData = new NodeData
{
CurrentTerm = message.Term + 10,
Log = new NodeLog()
};
var raftNode = Substitute.For <INode>();
raftNode.Data.Returns(nodeData);
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
var response = service.AppendEntries(message);
// Assert
response.Success.Should().BeFalse(
"because node cannot apply log " +
"if the leader term is less than the nodes term");
}
public void LogReplicatorCallsAppendEntriesPassingEncodedLog()
{
// Arrange
var encodedLog = BitConverter.GetBytes(100);
AppendEntriesRequest request = null;
var service = Substitute.For <IRaftService>();
service.When(x => x.AppendEntries(Arg.Any <AppendEntriesRequest>()))
.Do(x => request = x.Arg <AppendEntriesRequest>());
var peers = new List <PeerNode> {
new PeerNode()
};
var handler = new LogReplicator(peers);
var @event = TestEventFactory.GetCommandEvent(1L, encodedLog);
// Act
handler.Handle(@event);
// Assert
request.Should().NotBeNull();
request.Entries.Should().HaveCount(1);
request.Entries[0].Should().Equal(encodedLog);
}
/// <summary>
///
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request)
{
try
{
var json = JsonConvert.SerializeObject(request);
var content = new StringContent(json);
content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
var response = await _httpClient.PostAsync("/_raft/appendEntries", content);
if (response != null && response.IsSuccessStatusCode)
{
var responseContent = await response.Content.ReadAsStringAsync();
return(JsonConvert.DeserializeObject <AppendEntriesResponse>(responseContent));
}
return(new AppendEntriesResponse(request.Term, false));
}
catch (HttpRequestException requestEx)
{
_logger.LogError($"AppendEntries fail: {requestEx.Message}, serverUri: {_httpClient.BaseAddress}");
return(new AppendEntriesResponse(request.Term, false));
}
catch (Exception ex)
{
_logger.LogError(ex, $"AppendEntries fail: {ex.Message}, serverUri: {_httpClient.BaseAddress}");
return(new AppendEntriesResponse(request.Term, false));
}
}
private bool GrantSuccess(AppendEntriesRequest requestVote)
{
try
{
if (requestVote.Term < Peer.MyState.CurrentTerm)
{
return(false);
}
if (!EntryTermMatches(requestVote))
{
return(false);
}
DeleteConflictEntries(requestVote.PrevLogIndex);
if (requestVote.entries != null)
{
Peer.MyState.Log.AddRange(requestVote.entries);
}
if (requestVote.LeaderCommitIndex > Peer.MyState.CommittedIndex)
{
Peer.MyState.CommittedIndex = Math.Min(requestVote.LeaderCommitIndex, Peer.MyState.Log.Count - 1);
WriteLine(Peer.MyState.CommittedIndex);
}
}
catch (Exception exception)
{
WriteLine(exception.Message);
}
return(true);
}
public AppendEntriesResponse AppendEntry(AppendEntriesRequest requestVote)
{
var electTime = ElectionTimeout;
Election.Change(electTime, Timeout.Infinite);
lock (_lockObject)
{
Peer.MyState.ThisServerInfo.PersistedState.SavePeerState(Peer.MyState);
foreach (var peer in Peer.MyState.PeerlList)
{
peer.Value.Persist(Peer.MyState.ThisServerInfo);
}
var differentStateOrLeader = Peer.MyState.CurrentState != CurrentServerState.Follower ||
Peer.MyState.Leader != null && Peer.MyState.Leader.Id != requestVote.Leader.Id;
if (requestVote.Term > Peer.MyState.CurrentTerm || Peer.MyState.CurrentTerm <= requestVote.Term && differentStateOrLeader)
{
WriteLine($"Server {Peer.MyState.Id} is reverting to follower from {Peer.MyState.CurrentState}");
Peer.MyState.CurrentTerm = requestVote.Term;
Peer.MyState.CurrentState = CurrentServerState.Follower;
Peer.MyState.Leader = requestVote.Leader;
}
var success = GrantSuccess(requestVote);
return(new AppendEntriesResponse(Peer.MyState.CurrentTerm, success, Peer.MyState.ThisServerInfo.Id));
}
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
if (MaybeIgnoreFrequentRequestsIfServerDown(dest))
{
return;
}
LogStatus("append entries to " + dest, async() =>
{
var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId);
using (var request = CreateRequest(dest, requestUri, HttpMethods.Post))
{
var httpResponseMessage = await request.WriteAsync(() => new EntriesContent(req.Entries)).ConfigureAwait(false);
UpdateConnectionFailureCounts(dest, httpResponseMessage);
var reply = await httpResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var appendEntriesResponse = JsonConvert.DeserializeObject <AppendEntriesResponse>(reply);
SendToSelf(appendEntriesResponse);
}
});
}
public override async Task<AppendEntriesResult> AppendEntries(AppendEntriesRequest request)
{
if (SleepOnAppendEntries)
{
await Task.Delay(Timeout.Infinite);
}
return await base.AppendEntries(request);
}
public override async Task <AppendEntriesResult> AppendEntries(AppendEntriesRequest request)
{
if (SleepOnAppendEntries)
{
await Task.Delay(Timeout.Infinite);
}
return(await base.AppendEntries(request));
}
/// <summary>
/// Processes the given append entries request.
/// </summary>
/// <param name="request">AppendEntriesRequest</param>
void AppendEntries(AppendEntriesRequest request)
{
if (request.Term < this.CurrentTerm)
{
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false,
this.Id, request.ReceiverEndpoint));
}
else
{
if (request.PrevLogIndex > 0 &&
(this.Logs.Count < request.PrevLogIndex ||
this.Logs[request.PrevLogIndex - 1].Term != request.PrevLogTerm))
{
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm,
false, this.Id, request.ReceiverEndpoint));
}
else
{
if (request.Entries.Count > 0)
{
var currentIndex = request.PrevLogIndex + 1;
foreach (var entry in request.Entries)
{
if (this.Logs.Count < currentIndex)
{
this.Logs.Add(entry);
}
else if (this.Logs[currentIndex - 1].Term != entry.Term)
{
this.Logs.RemoveRange(currentIndex - 1, this.Logs.Count - (currentIndex - 1));
this.Logs.Add(entry);
}
currentIndex++;
}
}
if (request.LeaderCommit > this.CommitIndex &&
this.Logs.Count < request.LeaderCommit)
{
this.CommitIndex = this.Logs.Count;
}
else if (request.LeaderCommit > this.CommitIndex)
{
this.CommitIndex = request.LeaderCommit;
}
if (this.CommitIndex > this.LastApplied)
{
this.LastApplied++;
}
this.LeaderId = request.LeaderId;
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm,
true, this.Id, request.ReceiverEndpoint));
}
}
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
if (_linkedTokenSource.IsCancellationRequested)
{
return;
}
_sender.Send(dest, req);
}
public override void Handle(CommandScheduled @event)
{
var request = new AppendEntriesRequest {
Entries = new[] {
@event.EncodedEntry
}
};
foreach (var peerNode in _peers)
{
GetChannel(peerNode).AppendEntries(request);
}
}
private bool EntryTermMatches(AppendEntriesRequest requestVote)
{
if (requestVote.PrevLogIndex < 0 && Peer.MyState.Log.Count == 0)
{
return(requestVote.PrevLogIndex == -1);
}
if (requestVote.PrevLogIndex >= Peer.MyState.Log.Count)
{
return(false);
}
return(requestVote.PrevLogIndex == -1 || Peer.MyState.Log[requestVote.PrevLogIndex].Term == requestVote.PrevLogTerm);
}
public void PublishesToBufferAfterSafetyChecks()
{
// Arrange
var message = new AppendEntriesRequest
{
Term = 11,
PreviousLogIndex = 2,
PreviousLogTerm = 13,
LeaderCommit = 12,
Entries = new []
{
BitConverter.GetBytes(100),
BitConverter.GetBytes(200)
}
};
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Follower);
var nodeData = new NodeData {
Log = new NodeLog()
};
nodeData.Log.SetLogEntry(message.PreviousLogIndex, message.PreviousLogTerm);
raftNode.Data.Returns(nodeData);
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = new TestBufferPublisher <NodeCommandScheduled, NodeCommandResult>();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
service.AppendEntries(message);
// Assert
appendEntriesPublisher.Received()
.PublishEvent(Arg.Is <AppendEntriesRequested>(requested =>
requested.PreviousLogIndex == message.PreviousLogIndex &&
requested.PreviousLogTerm == message.PreviousLogTerm &&
requested.LeaderCommit == message.LeaderCommit &&
requested.Entries == message.Entries));
}
public override AppendEntriesResponse Handle(AppendEntriesRequest req)
{
if (_installingSnapshot == null)
{
return(base.Handle(req));
}
var lastLogEntry = Engine.PersistentState.LastLogEntry();
return(new AppendEntriesResponse
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogEntry.Index,
LeaderId = Engine.CurrentLeader,
Message = "I am in the process of receiving a snapshot, so I cannot accept new entries at the moment",
Success = false
});
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("append entries to " + dest, async() =>
{
var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.PostAsync(requestUri, new EntriesContent(req.Entries));
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var appendEntriesResponse = JsonConvert.DeserializeObject <AppendEntriesResponse>(reply);
SendToSelf(appendEntriesResponse);
});
}
}
public void AppendEntriesResetsTimer()
{
// Arrange
var message = new AppendEntriesRequest();
var raftNode = Substitute.For <INode>();
raftNode.CurrentState.Returns(NodeState.Follower);
raftNode.Data.Returns(new NodeData {
Log = new NodeLog()
});
var timer = Substitute.For <INodeTimer>();
var appendEntriesPublisher = Substitute.For <IPublishToBuffer <AppendEntriesRequested> >();
var nodePublisher = Substitute.For <IPublishToBuffer <NodeCommandScheduled, NodeCommandResult> >();
var service = new RaftService(appendEntriesPublisher, nodePublisher, timer, raftNode);
// Act
service.AppendEntries(message);
// Assert
timer.Received(1).ResetTimer();
}
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req)
{
if (FromOurTopology(req) == false)
{
_log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.CurrentLeader,
Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
if (req.Term < Engine.PersistentState.CurrentTerm)
{
var msg = string.Format(
"Rejecting append entries because msg term {0} is lower then current term: {1}",
req.Term, Engine.PersistentState.CurrentTerm);
_log.Info(msg);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.CurrentLeader,
Message = msg,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
if (req.Term > Engine.PersistentState.CurrentTerm)
{
Engine.UpdateCurrentTerm(req.Term, req.From);
}
if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false)
{
Engine.CurrentLeader = req.From;
Engine.SetState(RaftEngineState.Follower);
}
var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0;
if (prevTerm != req.PrevLogTerm)
{
var msg = string.Format(
"Rejecting append entries because msg previous term {0} is not the same as the persisted current term {1} at log index {2}",
req.PrevLogTerm, prevTerm, req.PrevLogIndex);
_log.Info(msg);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
Message = msg,
LeaderId = Engine.CurrentLeader,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
LastHeartbeatTime = DateTime.UtcNow;
LastMessageTime = DateTime.UtcNow;
if (req.Entries.Length > 0)
{
_log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length,
Engine.State);
foreach (var logEntry in req.Entries)
{
_log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term);
}
// if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat
// to process a message). In this case, our log already have the entries in question, and it would be a waste to
// truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term
// values in our log and in the entries, and then skip over the duplicates.
var skip = 0;
for (int i = 0; i < req.Entries.Length; i++)
{
var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1;
if (termForEntry != req.Entries[i].Term)
{
break;
}
skip++;
}
if (skip != req.Entries.Length)
{
Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip);
}
var topologyChange = req.Entries.LastOrDefault(x => x.IsTopologyChange == true);
// we consider the latest topology change to be in effect as soon as we see it, even before the
// it is committed, see raft spec section 6:
// a server always uses the latest configuration in its log,
// regardless of whether the entry is committed
if (topologyChange != null)
{
var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data);
var topologyChangeCommand = command as TopologyChangeCommand;
if (topologyChangeCommand == null) //precaution,should never be true
//if this is true --> it is a serious issue and should be fixed immediately!
{
throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand.
Instead, it is of type: " + command.GetType() + ". It is probably a bug!");
}
_log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}",
topologyChangeCommand.Requested);
Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index);
Engine.StartTopologyChange(topologyChangeCommand);
}
}
var lastIndex = req.Entries.Length == 0 ?
Engine.PersistentState.LastLogEntry().Index :
req.Entries[req.Entries.Length - 1].Index;
try
{
var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex);
if (nextCommitIndex > Engine.CommitIndex)
{
CommitEntries(req.Entries, nextCommitIndex);
}
return(new AppendEntriesResponse
{
Success = true,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
catch (Exception e)
{
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
Message = "Failed to apply new entries. Reason: " + e,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
}
public virtual AppendEntriesResponse Handle(AppendEntriesRequest req)
{
var lastLogIndex = Engine.PersistentState.LastLogEntry().Index;
if (FromOurTopology(req) == false)
{
_log.Info("Got an append entries message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogIndex,
LeaderId = Engine.CurrentLeader,
Message = "Cannot accept append entries from a node outside my cluster. My topology id is: " + Engine.CurrentTopology.TopologyId,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
if (req.Term < Engine.PersistentState.CurrentTerm)
{
var msg = string.Format(
"Rejecting append entries because msg term {0} is lower then current term: {1}",
req.Term, Engine.PersistentState.CurrentTerm);
_log.Info(msg);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogIndex,
LeaderId = Engine.CurrentLeader,
Message = msg,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
if (req.Term > Engine.PersistentState.CurrentTerm)
{
Engine.UpdateCurrentTerm(req.Term, req.From);
}
if (Engine.CurrentLeader == null || req.From.Equals(Engine.CurrentLeader) == false)
{
Engine.CurrentLeader = req.From;
Engine.SetState(RaftEngineState.Follower);
}
var prevTerm = Engine.PersistentState.TermFor(req.PrevLogIndex) ?? 0;
if (prevTerm != req.PrevLogTerm)
{
var midpointIndex = req.PrevLogIndex / 2;
var midpointTerm = Engine.PersistentState.TermFor(midpointIndex) ?? 0;
var msg = $"Rejecting append entries because msg previous term {req.PrevLogTerm} is not the same as the persisted current term {prevTerm}" +
$" at log index {req.PrevLogIndex}. Midpoint index {midpointIndex}, midpoint term: {midpointTerm}";
_log.Info(msg);
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = req.PrevLogIndex,
Message = msg,
LeaderId = Engine.CurrentLeader,
MidpointIndex = midpointIndex,
MidpointTerm = midpointTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
LastHeartbeatTime = DateTime.UtcNow;
LastMessageTime = DateTime.UtcNow;
var appendEntriesResponse = new AppendEntriesResponse
{
Success = true,
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
if (req.Entries.Length > 0)
{
if (_log.IsDebugEnabled)
{
_log.Debug("Appending log (persistant state), entries count: {0} (node state = {1})", req.Entries.Length,
Engine.State);
foreach (var logEntry in req.Entries)
{
_log.Debug("Entry {0} (term {1})", logEntry.Index, logEntry.Term);
}
}
// if is possible that we'll get the same event multiple times (for example, if we took longer than a heartbeat
// to process a message). In this case, our log already have the entries in question, and it would be a waste to
// truncate the log and re-add them all the time. What we are doing here is to find the next match for index/term
// values in our log and in the entries, and then skip over the duplicates.
var skip = 0;
for (int i = 0; i < req.Entries.Length; i++)
{
var termForEntry = Engine.PersistentState.TermFor(req.Entries[i].Index) ?? -1;
if (termForEntry != req.Entries[i].Term)
{
break;
}
skip++;
}
var topologyChange = req.Entries.Skip(skip).LastOrDefault(x => x.IsTopologyChange == true);
if (topologyChange != null)
{
var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data);
var topologyChangeCommand = command as TopologyChangeCommand;
if (topologyChangeCommand != null && topologyChangeCommand.Requested.AllNodes.Select(x => x.Name).Contains(Engine.Options.SelfConnection.Name) == false)
{
_log.Warn("Got topology without self, disconnecting from the leader, clearing topology and moving to leader state");
var tcc = new TopologyChangeCommand
{
Requested = new Topology(Guid.NewGuid(), new[] { Engine.Options.SelfConnection }, new List <NodeConnectionInfo>(), new List <NodeConnectionInfo>())
};
Engine.PersistentState.SetCurrentTopology(tcc.Requested, 0L);
Engine.StartTopologyChange(tcc);
Engine.CommitTopologyChange(tcc);
Engine.SetState(RaftEngineState.Leader);
return(new AppendEntriesResponse
{
Success = true,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogIndex,
Message = "Leaving cluster, because received topology from the leader that didn't contain us",
From = Engine.Name,
ClusterTopologyId = req.ClusterTopologyId, // we send this "older" ID, so the leader won't reject us
});
}
}
if (skip != req.Entries.Length)
{
Engine.PersistentState.AppendToLog(Engine, req.Entries.Skip(skip), req.PrevLogIndex + skip);
}
else
{
// if we skipped the whole thing, this is fine, but let us hint to the leader that we are more
// up to date then it thinks
var lastReceivedIndex = req.Entries[req.Entries.Length - 1].Index;
appendEntriesResponse.MidpointIndex = lastReceivedIndex + (lastLogIndex - lastReceivedIndex) / 2;
appendEntriesResponse.MidpointTerm = Engine.PersistentState.TermFor(appendEntriesResponse.MidpointIndex.Value) ?? 0;
_log.Info($"Got {req.Entries.Length} entires from index {req.Entries[0].Index} with term {req.Entries[0].Term} skipping all. " +
$"Setting midpoint index to {appendEntriesResponse.MidpointIndex} with term {appendEntriesResponse.MidpointTerm}.");
}
// we consider the latest topology change to be in effect as soon as we see it, even before the
// it is committed, see raft spec section 6:
// a server always uses the latest con?guration in its log,
// regardless of whether the entry is committed
if (topologyChange != null)
{
var command = Engine.PersistentState.CommandSerializer.Deserialize(topologyChange.Data);
var topologyChangeCommand = command as TopologyChangeCommand;
if (topologyChangeCommand == null) //precaution,should never be true
//if this is true --> it is a serious issue and should be fixed immediately!
{
throw new InvalidOperationException(@"Log entry that is marked with IsTopologyChange should be of type TopologyChangeCommand.
Instead, it is of type: " + command.GetType() + ". It is probably a bug!");
}
_log.Info("Topology change started (TopologyChangeCommand committed to the log): {0}",
topologyChangeCommand.Requested);
Engine.PersistentState.SetCurrentTopology(topologyChangeCommand.Requested, topologyChange.Index);
Engine.StartTopologyChange(topologyChangeCommand);
}
}
var lastIndex = req.Entries.Length == 0 ?
lastLogIndex :
req.Entries[req.Entries.Length - 1].Index;
try
{
var nextCommitIndex = Math.Min(req.LeaderCommit, lastIndex);
if (nextCommitIndex > Engine.CommitIndex)
{
CommitEntries(req.Entries, nextCommitIndex);
}
appendEntriesResponse.LastLogIndex = lastLogIndex;
return(appendEntriesResponse);
}
catch (Exception e)
{
return(new AppendEntriesResponse
{
Success = false,
CurrentTerm = Engine.PersistentState.CurrentTerm,
LastLogIndex = lastLogIndex,
Message = "Failed to apply new entries. Reason: " + e,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
});
}
}
/// <summary>
/// Processes the given append entries request.
/// </summary>
/// <param name="request">AppendEntriesRequest</param>
void AppendEntries(AppendEntriesRequest request)
{
if (request.Term < this.CurrentTerm)
{
Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " +
this.Logs.Count + " | last applied: " + this.LastApplied + " | append false (< term)\n");
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm, false,
this.Id, request.ReceiverEndpoint));
}
else
{
if (request.PrevLogIndex > 0 &&
(this.Logs.Count < request.PrevLogIndex ||
this.Logs[request.PrevLogIndex - 1].Term != request.PrevLogTerm))
{
Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " +
this.Logs.Count + " | last applied: " + this.LastApplied + " | append false (not in log)\n");
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm,
false, this.Id, request.ReceiverEndpoint));
}
else
{
if (request.Entries.Count > 0)
{
var currentIndex = request.PrevLogIndex + 1;
foreach (var entry in request.Entries)
{
if (this.Logs.Count < currentIndex)
{
this.Logs.Add(entry);
}
else if (this.Logs[currentIndex - 1].Term != entry.Term)
{
this.Logs.RemoveRange(currentIndex - 1, this.Logs.Count - (currentIndex - 1));
this.Logs.Add(entry);
}
currentIndex++;
}
}
if (request.LeaderCommit > this.CommitIndex &&
this.Logs.Count < request.LeaderCommit)
{
this.CommitIndex = this.Logs.Count;
}
else if (request.LeaderCommit > this.CommitIndex)
{
this.CommitIndex = request.LeaderCommit;
}
if (this.CommitIndex > this.LastApplied)
{
this.LastApplied++;
}
Console.WriteLine("\n [Server] " + this.ServerId + " | term " + this.CurrentTerm + " | log " +
this.Logs.Count + " | entries received " + request.Entries.Count + " | last applied " +
this.LastApplied + " | append true\n");
this.LeaderId = request.LeaderId;
this.Send(request.LeaderId, new AppendEntriesResponse(this.CurrentTerm,
true, this.Id, request.ReceiverEndpoint));
}
}
}
public async Task <AppendEntriesResult> AppendEntries([FromBody] AppendEntriesRequest request)
{
return(await node.AppendEntries(request));
}
/// <summary>
///
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public async Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request)
{
return(await _rpcClient.AppendEntriesAsync(request));
}
private void SendEntriesToPeer(NodeConnectionInfo peer)
{
LogEntry prevLogEntry;
LogEntry[] entries;
var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value
if (Engine.StateMachine.SupportSnapshots && nextIndex != -1)
{
var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex();
if (snapshotIndex != null && nextIndex < snapshotIndex)
{
if (_snapshotsPendingInstallation.ContainsKey(peer.Name))
{
return;
}
var snapshotWriter = Engine.StateMachine.GetSnapshotWriter();
Engine.Transport.Send(peer, new CanInstallSnapshotRequest
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
Index = snapshotWriter.Index,
Term = snapshotWriter.Term,
});
return;
}
}
if (nextIndex == -1)
{
nextIndex = 0;
}
try
{
entries = Engine.PersistentState.LogEntriesAfter(nextIndex)
.Take(Engine.Options.MaxEntriesPerRequest)
.ToArray();
prevLogEntry = entries.Length == 0
? Engine.PersistentState.LastLogEntry()
: Engine.PersistentState.GetLogEntry(entries[0].Index - 1);
}
catch (Exception e)
{
_log.Error("Error while fetching entries from persistent state.", e);
throw;
}
if (_log.IsDebugEnabled)
{
_log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index);
}
var aer = new AppendEntriesRequest
{
Entries = entries,
LeaderCommit = Engine.CommitIndex,
PrevLogIndex = prevLogEntry.Index,
PrevLogTerm = prevLogEntry.Term,
Term = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
Engine.Transport.Send(peer, aer);
Engine.OnEntriesAppended(entries);
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
_sender.Send(dest, req);
}
