public async Task <SendLeaderCommandResponse> Receive(ICommand command)
{
if (State is Follower)
{
await _messageBus.Send(command, LeaderId);
}
if (State is Leader)
{
_logger.LogDebug("Server Received Command");
_appendingEntries = true;
Log.Add(new Log(CurrentTerm, command));
CommitIndex = Log.Count - 1;
var remoteServers = GetRemoteServers();
var tasks = new Task <AppendEntriesResponse> [remoteServers.Count];
for (int i = 0; i < tasks.Length; i++)
{
var next = NextIndex.FirstOrDefault(x => x.Id == remoteServers[i].Id);
if (next == null)
{
var nextLogIndex = 0;
next = new Next(remoteServers[i].Id, nextLogIndex);
NextIndex.Add(next);
}
var match = MatchIndex.FirstOrDefault(x => x.Id == remoteServers[i].Id);
if (match == null)
{
match = new Match(remoteServers[i].Id, 0);
MatchIndex.Add(match);
}
var lastLogIndex = Log.Count > 0 ? Log.Count - 1 : 0;
var lastLogTerm = lastLogIndex > 0 ? Log[match.MatchIndex].Term : 0;
// If last log index ≥ nextIndex for a follower: send
// AppendEntries RPC with log entries starting at nextIndex
if (lastLogIndex >= next.NextIndex)
{
var log = Log[next.NextIndex];
var appendEntries = new AppendEntries(CurrentTerm, Id, match.MatchIndex, lastLogTerm, log, CommitIndex, remoteServers[i].Id);
tasks[i] = _messageBus.Send(appendEntries);
}
}
Task.WaitAll(tasks);
int counter = 0;
foreach (var task in tasks)
{
_logger.LogDebug($"Processing Append entries counter: {counter}");
_logger.LogDebug($"Processing Append entries result was: {task.Result.Success} counter: {counter}");
await Receive(task.Result);
_logger.LogDebug($"Processed Append entries counter: {counter}");
}
}
return(new SendLeaderCommandResponse());
}
public AppendEntriesResponse Request(AppendEntries appendEntries)
{
try
{
if (_token == null)
{
SetToken();
}
var json = JsonConvert.SerializeObject(appendEntries, _jsonSerializerSettings);
var content = new StringContent(json);
content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
var response = _httpClient.PostAsync($"{_hostAndPort}/administration/raft/appendEntries", content).GetAwaiter().GetResult();
if (response.IsSuccessStatusCode)
{
return(JsonConvert.DeserializeObject <AppendEntriesResponse>(response.Content.ReadAsStringAsync().GetAwaiter().GetResult(), _jsonSerializerSettings));
}
else
{
return(new AppendEntriesResponse(appendEntries.Term, false));
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(new AppendEntriesResponse(appendEntries.Term, false));
}
}
public void Receive(SendHeartbeat sendHeartbeat)
{
if (State is Leader)
{
var remoteServers = GetRemoteServers();
var tasks = new Task <AppendEntriesResponse> [remoteServers.Count];
for (int i = 0; i < tasks.Length; i++)
{
var lastLogIndex = Log.Count > 0 ? Log.Count - 1 : 0;
var lastLogTerm = lastLogIndex > 0 ? Log[lastLogIndex].Term : 0;
var appendEntries = new AppendEntries(CurrentTerm, Id, lastLogIndex, lastLogTerm, null, CommitIndex, remoteServers[i].Id);
tasks[i] = _messageBus.Send(appendEntries);
}
Task.WaitAll(tasks);
foreach (var task in tasks)
{
if (task.Result.Term > CurrentTerm)
{
BecomeFollowerAndMatchTerm(task.Result.Term, task.Result.FollowerId);
}
}
_messageBus.Publish(new SendToSelf(new SendHeartbeat()));
}
}
public void NodeStaysAFollowerWhenReceiveAppendEntries()
{
using (Helpers.InitLog4Net())
{
var settings = Helpers.BuildNodeSettings("1", new[] { "1", "2", "3", "4", "5" });
settings.TimeoutInMs = 20;
var middleware = new Middleware();
var node = new Node <string>(TestHelpers.GetPool().BuildSequencer(), settings, middleware, new StateMachine());
using (node)
{
node.Initialize();
// should switch to candidate
Check.That(this.WaitState(node, NodeStatus.Candidate, 40)).IsTrue();
// now we pretend there is a leader
var message = new AppendEntries <string>
{
LeaderId = "2",
LeaderTerm = 5,
PrevLogIndex = -1,
PrevLogTerm = 0
};
var entry = new LogEntry <string>("dummy", 1L);
message.Entries = new[] { entry };
middleware.SendMessage("1", message);
Check.That(this.WaitState(node, NodeStatus.Follower, 30)).IsTrue();
Check.That(node.State.LogEntries.Count).IsEqualTo(1);
}
}
}
public async Task <AppendEntriesResponse> Send(AppendEntries appendEntries)
{
try
{
var serverToSendMessageTo = _serviceRegistry.Get(RaftyServiceDiscoveryName.Get()).First(x => x.Id == appendEntries.FollowerId);
var json = JsonConvert.SerializeObject(appendEntries, Formatting.None, new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.All
});
var httpContent = new StringContent(json);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
using (var httpClient = new HttpClient())
{
httpClient.BaseAddress = serverToSendMessageTo.Location;
var response = await httpClient.PostAsync(_appendEntriesUrl, httpContent);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var appendEntriesResponse = JsonConvert.DeserializeObject <AppendEntriesResponse>(content);
return(appendEntriesResponse);
}
}
catch (Exception exception)
{
_logger.LogError(new EventId(1), exception, "Error in Send(AppendEntries appendEntries)");
throw;
}
}
private void GivenTheCandidatesLogIsAtIndex(int index)
{
var entries = new Log(1, new FakeCommand(Guid.NewGuid()));
var appendEntries = new AppendEntries(1, Guid.NewGuid(), index, 0, entries, 0, Guid.NewGuid());
_server.Receive(appendEntries).Wait();
}
public async Task <AppendEntriesResponse> Request(AppendEntries appendEntries)
{
if (appendEntries.Entries.Count > 0)
{
AppendEntriesResponsesWithLogEntries++;
}
AppendEntriesResponses++;
return(_appendEntriesResponse);
}
public void should_add_remote_server_if_receive_append_entries_from_unknown_remote_server()
{
var appendEntries = new AppendEntries(20, Guid.NewGuid(), 0, 0, null, 0, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => TheRemoteServerCountIs(1))
.BDDfy();
}
public Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
//follower不对外输出日志
return(Task.FromResult(new AppendEntriesResponse()
{
Success = false,
Term = CurrentState.CurrentTerm
}));
}
public void server_should_become_follower_if_receives_greater_term_in_append_entries_message()
{
var appendEntries = new AppendEntries(20, Guid.NewGuid(), 0, 0, null, 0, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheCurrentTermIs(20))
.And(x => TheServerIsAFollower())
.BDDfy();
}
public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
var response = await State.Handle(appendEntries);
if (response.BlockEntity != null)
{
_logger.LogInformation($"{State.GetType().Name} id: {State.CurrentState.Id} responded to appendentries with success: {response.Success} and term: {response.Term}");
}
return(response);
}
private void AppendEntriesTermIsGreaterThanCurrentTerm(AppendEntries appendEntries)
{
if (appendEntries.Term > CurrentState.CurrentTerm)
{
CurrentState = new CurrentState(CurrentState.Id, appendEntries.Term, CurrentState.VotedFor,
CurrentState.CommitIndex, CurrentState.LastApplied, CurrentState.LeaderId);
BecomeFollower();
}
}
//追加 peer
public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
var node = _nodePeer.GetNode(appendEntries.ChannelId);
if (node == null)
{
throw new Exception($"节点未加入{appendEntries.ChannelId} 通道");
}
return(await node.Handle(appendEntries));
}
public async Task <AppendEntriesResponse> Request(AppendEntries appendEntries)
{
try
{
return(await _node.Handle(appendEntries));
}
catch (Exception e)
{
return(new AppendEntriesResponse(0, false));
}
}
public void server_should_become_follower_if_receives_append_entries_while_candidate()
{
var appendEntries = new AppendEntries(0, Guid.NewGuid(), 0, 0, null, 0, Guid.NewGuid());
this.Given(x => GivenANewServer())
.And(x => ServerReceives(new BecomeCandidate(Guid.NewGuid())))
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheServerIsAFollower())
.And(x => ThenTheCurrentTermVotesAre(0))
.BDDfy();
}
public AppendEntriesResponse Request(AppendEntries appendEntries)
{
try
{
return(_node.Handle(appendEntries));
}
catch (Exception e)
{
return(new AppendEntriesResponse(0, false));
}
}
public void server_should_set_current_term_as_message_term_if_greater_than_current_term()
{
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(1, id, 0, 0, null, 0, Guid.NewGuid());
var expected = new AppendEntriesResponse(1, true, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheReplyIs(expected))
.And(x => ThenTheCurrentTermIs(1))
.BDDfy();
}
public void server_should_reply_true_if_entries_is_empty()
{
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(0, id, 0, 0, null, 0, Guid.NewGuid());
var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheReplyIs(expected))
.And(x => ThenTheCurrentTermIs(0))
.BDDfy();
}
public void server_should_append_new_entries_not_in_log_and_reply_true()
{
var entries = new Log(0, new FakeCommand(Guid.NewGuid()));
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(0, id, 1, 0, entries, 0, Guid.NewGuid());
var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheReplyIs(expected))
.And(x => ThenTheLogContainsEntriesCount(1))
.BDDfy();
}
public void server_should_reply_false_if_term_is_less_than_current_term()
{
var entries = new Log(1, new FakeCommand(Guid.NewGuid()));
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(0, id, 0, 0, entries, 0, Guid.NewGuid());
var expected = new AppendEntriesResponse(1, false, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.And(x => GivenTheCurrentTermIs(1))
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheReplyIs(expected))
.BDDfy();
}
public void should_set_commit_index_if_leader_commit_greater_than_commit_index()
{
var entries = new Log(0, new FakeCommand(Guid.NewGuid()));
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(0, id, 1, 0, entries, 1, Guid.NewGuid());
var expected = new AppendEntriesResponse(0, true, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.When(x => ServerReceives(appendEntries))
.Then(x => ThenTheReplyIs(expected))
.And(x => ThenTheCommitIndexIs(1))
.BDDfy();
}
public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
try
{
await _appendingEntries.WaitAsync();
var response = _rules.AppendEntriesTermIsLessThanCurrentTerm(appendEntries, CurrentState);
if (response.shouldReturn)
{
return(response.appendEntriesResponse);
}
response =
await _rules.LogDoesntContainEntryAtPreviousLogIndexWhoseTermMatchesPreviousLogTerm(appendEntries,
_log, CurrentState);
if (response.shouldReturn)
{
return(response.appendEntriesResponse);
}
await _rules.DeleteAnyConflictsInLog(appendEntries, _log);
var terms = appendEntries.Entries.Any()
? string.Join(",", appendEntries.Entries.Select(x => x.Term))
: string.Empty;
_logger.LogInformation(
$"{CurrentState.Id} as {nameof(Follower)} applying {appendEntries.Entries.Count} to log, term {terms}");
await _rules.ApplyNewEntriesToLog(appendEntries, _log);
var commitIndexAndLastApplied =
await _rules.CommitIndexAndLastApplied(appendEntries, _log, CurrentState);
await ApplyToStateMachine(commitIndexAndLastApplied.commitIndex, commitIndexAndLastApplied.lastApplied,
appendEntries);
SetLeaderId(appendEntries);
_messagesSinceLastElectionExpiry++;
return(new AppendEntriesResponse(CurrentState.CurrentTerm, true));
}
finally
{
_appendingEntries.Release();
}
}
internal override void ProcessAppendEntries(AppendEntries <T> appendEntries)
{
if (appendEntries.LeaderTerm >= this.CurrentTerm)
{
this.Logger.InfoFormat(
"Received AppendEntries from a probable leader, stepping down ({0}).",
appendEntries);
this.Node.SwitchToAndProcessMessage(NodeStatus.Follower, appendEntries);
}
else
{
Logger.Debug("Received AppendEntries from an invalid leader, refusing.");
var reply = new AppendEntriesAck(this.Node.Id, this.CurrentTerm, false);
this.Node.SendMessage(appendEntries.LeaderId, reply);
}
}
public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
if (appendEntries.Term > CurrentState.CurrentTerm)
{
var response = await _rules.CommitIndexAndLastApplied(appendEntries, _log, CurrentState);
await ApplyToStateMachine(appendEntries, response.commitIndex, response.lastApplied);
SetLeaderId(appendEntries);
_node.BecomeFollower(CurrentState);
return(new AppendEntriesResponse(CurrentState.CurrentTerm, true));
}
return(new AppendEntriesResponse(CurrentState.CurrentTerm, false));
}
internal override void ProcessAppendEntries(AppendEntries <T> appendEntries)
{
bool result;
if (appendEntries.LeaderTerm < this.CurrentTerm)
{
// leader is older than us or log does not match
this.Logger.DebugFormat(
"Reject an AppendEntries from an invalid leader ({0}).", appendEntries);
result = false;
}
else
{
// we will proceed
this.Node.LeaderId = appendEntries.LeaderId;
if (appendEntries.LeaderTerm > this.CurrentTerm)
{
this.Logger.TraceFormat("Upgrade our term to {0}.", this.CurrentTerm);
this.Node.State.CurrentTerm = appendEntries.LeaderTerm;
}
if (this.Node.State.EntryMatches(
appendEntries.PrevLogIndex, appendEntries.PrevLogTerm))
{
this.Logger.TraceFormat(
"Process an AppendEntries request: {0}", appendEntries);
this.Node.State.AppendEntries(appendEntries.PrevLogIndex, appendEntries.Entries);
this.Node.Commit(appendEntries.CommitIndex);
result = true;
}
else
{
// log does not match, we are not in sync with leader yet
this.Logger.DebugFormat(
"Reject an AppendEntries that does not match our log ({0}).",
appendEntries);
result = false;
}
}
var reply = new AppendEntriesAck(this.Node.Id, this.CurrentTerm, result);
this.Node.SendMessage(appendEntries.LeaderId, reply);
this.ResetTimeout(.2);
}
server_should_reply_false_if_log_doesnt_contain_an_entry_at_previous_log_index_matching_previous_log_term()
{
var entries = new Log(0, new FakeCommand(Guid.NewGuid()));
var id = Guid.NewGuid();
var appendEntries = new AppendEntries(0, id, 0, 0, entries, 0, Guid.NewGuid());
var oldAppendEntries = new AppendEntries(0, id, 0, 1, entries, 0, Guid.NewGuid());
var expected = new AppendEntriesResponse(0, false, id, Guid.NewGuid());
this.Given(x => GivenANewServer())
.And(x => GivenTheServerRecieves(appendEntries))
.When(x => ServerReceives(oldAppendEntries))
.Then(x => ThenTheReplyIs(expected))
.BDDfy();
}
public void Send(JsonOperationContext context, Action updateFollowerTicks, AppendEntries ae, List <BlittableJsonReaderObject> items = null)
{
if (_log.IsInfoEnabled)
{
if (ae.EntriesCount > 0)
{
_log.Info(
$"AppendEntries ({ae.EntriesCount:#,#;;0}) in {ae.Term:#,#;;0}, commit: {ae.LeaderCommit:#,#;;0}, leader for: {ae.TimeAsLeader:#,#;;0}, ({ae.PrevLogIndex:#,#;;0} / {ae.PrevLogTerm:#,#;;0}), truncate: {ae.TruncateLogBefore:#,#;;0}, force elections: {ae.ForceElections}.");
}
}
var msg = new DynamicJsonValue
{
["Type"] = nameof(AppendEntries),
[nameof(AppendEntries.EntriesCount)] = ae.EntriesCount,
[nameof(AppendEntries.LeaderCommit)] = ae.LeaderCommit,
[nameof(AppendEntries.PrevLogIndex)] = ae.PrevLogIndex,
[nameof(AppendEntries.PrevLogTerm)] = ae.PrevLogTerm,
[nameof(AppendEntries.Term)] = ae.Term,
[nameof(AppendEntries.TruncateLogBefore)] = ae.TruncateLogBefore,
[nameof(AppendEntries.TimeAsLeader)] = ae.TimeAsLeader,
[nameof(AppendEntries.SendingThread)] = Thread.CurrentThread.ManagedThreadId,
[nameof(AppendEntries.MinCommandVersion)] = ae.MinCommandVersion
};
if (ae.ForceElections)
{
msg[nameof(AppendEntries.ForceElections)] = true;
}
Send(context, msg);
if (items == null || items.Count == 0)
{
return;
}
foreach (var item in items)
{
updateFollowerTicks();
Send(context, item);
}
}
public async Task <AppendEntriesResponse> Handle(AppendEntries appendEntries)
{
try
{
await _appendingEntries.WaitAsync();
var response = await State.Handle(appendEntries);
if (appendEntries.Entries.Any())
{
_logger.LogInformation($"{State.GetType().Name} id: {State.CurrentState.Id} responded to appendentries with success: {response.Success} and term: {response.Term}");
}
return(response);
}
finally
{
_appendingEntries.Release();
}
}
private async Task ApplyToStateMachine(int commitIndex, int lastApplied, AppendEntries appendEntries)
{
while (commitIndex > lastApplied)
{
_logger.LogInformation($"id: {CurrentState.Id} handling log in fsm in loop, commitIndex: {commitIndex}, lastApplied: {lastApplied}, ae.Count: {appendEntries.Entries.Count}");
lastApplied++;
var log = await _log.Get(lastApplied);
await _fsm.Handle(log);
}
CurrentState = new CurrentState(CurrentState.Id, appendEntries.Term,
CurrentState.VotedFor, commitIndex, lastApplied, CurrentState.LeaderId);
_logger.LogInformation($"id: {CurrentState.Id} handling log in fsm out of loop now, commitIndex: {commitIndex}, lastApplied: {lastApplied}, ae.Count: {appendEntries.Entries.Count}");
_logger.LogInformation($"id: {CurrentState.Id} CurrentState.CommitIndex: {CurrentState.CommitIndex}, CurrentState.LastApplied: {CurrentState.LastApplied}");
}
private void BecomeLeader()
{
if (State is Candidate)
{
State = new Leader();
LeaderId = Id;
_messageBus.Publish(new SendToSelf(new SendHeartbeat()));
NextIndex = new List <Next>();
MatchIndex = new List <Match>();
var remoteServers = GetRemoteServers();
for (var i = 0; i < remoteServers.Count; i++)
{
var nextLogIndex = Log.Count;
var next = new Next(remoteServers[i].Id, nextLogIndex);
NextIndex.Add(next);
var match = new Match(remoteServers[i].Id, 0);
MatchIndex.Add(match);
}
var tasks = new Task <AppendEntriesResponse> [remoteServers.Count];
for (var i = 0; i < tasks.Length; i++)
{
var lastLogIndex = Log.Count > 0 ? Log.Count - 1 : 0;
var lastLogTerm = lastLogIndex > 0 ? Log[lastLogIndex].Term : 0;
var appendEntries = new AppendEntries(CurrentTerm, Id, lastLogIndex, lastLogTerm, null, CommitIndex, remoteServers[i].Id);
tasks[i] = _messageBus.Send(appendEntries);
}
Task.WaitAll(tasks);
foreach (var task in tasks)
{
if (task.Result.Term > CurrentTerm)
{
BecomeFollowerAndMatchTerm(task.Result.Term, task.Result.FollowerId);
}
}
}
}
public void NodeStaysAFollowerWhenReceiveAppendEntries()
{
using (Helpers.InitLog4Net())
{
var settings = Helpers.BuildNodeSettings("1", new[] { "1", "2", "3", "4", "5" });
settings.TimeoutInMs = 20;
var middleware = new Middleware();
var node = new Node<string>(TestHelpers.GetPool().BuildSequencer(), settings, middleware, new StateMachine());
using (node)
{
node.Initialize();
// should switch to candidate
Check.That(this.WaitState(node, NodeStatus.Candidate, 40)).IsTrue();
// now we pretend there is a leader
var message = new AppendEntries<string>
{
LeaderId = "2",
LeaderTerm = 5,
PrevLogIndex = -1,
PrevLogTerm = 0
};
var entry = new LogEntry<string>("dummy", 1L);
message.Entries = new[] { entry };
middleware.SendMessage("1", message);
Check.That(this.WaitState(node, NodeStatus.Follower, 30)).IsTrue();
Check.That(node.State.LogEntries.Count).IsEqualTo(1);
}
}
}