public void ShouldBeAbleToGetLastTimestampOfLatestHeartbeat()
{
// The first response should be DateTime.Never
var requesterId = Guid.NewGuid();
var response = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(response);
Assert.IsType <DateTime>(response.ResponseMessage);
var lastUpdated = (DateTime)response.ResponseMessage;
Assert.True(lastUpdated.Equals(default(DateTime)));
Assert.True(lastUpdated.Equals(DateTime.MinValue));
var currentTime = DateTime.UtcNow;
var appendEntries = new Request <AppendEntries>(requesterId,
new AppendEntries(0, Guid.NewGuid(), 0, 0, new object[0], 0));
_raftNode.Tell(appendEntries);
var secondResponse = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(secondResponse);
Assert.IsType <DateTime>(secondResponse.ResponseMessage);
// The last updated timestamp should be relatively similar to the current time
var timestamp = (DateTime)secondResponse.ResponseMessage;
var timeDifference = timestamp - currentTime;
Assert.True(timeDifference.TotalMilliseconds >= 0 && timeDifference <= TimeSpan.FromSeconds(5));
}
public void ShouldBecomeFollowerIfAppendEntriesReceivedFromNewLeader()
{
var electionTimeoutInMilliseconds = 1000;
var initialTerm = 0;
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var nodeId = Guid.NewGuid();
var numberOfOtherActors = 2;
IEnumerable <Guid> getOtherActors()
{
return(Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid()));
}
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors, initialTerm,
electionTimeoutInMilliseconds);
void BlockUntilTrue(Func <bool> condition)
{
while (!condition())
{
}
}
// Start the node and let it timeout to trigger an election
_raftNode.Tell(new Initialize());
eventLog.BlockUntilAny(msg => msg is RoleStateChanged rsc && rsc.Term == initialTerm);
// Look for the change in state to a candidate node
Assert.True(eventLog.Count(msg => msg is RoleStateChanged rsc &&
rsc.NewState == RoleState.Candidate &&
rsc.Term == initialTerm) == 1);
// Send a single heartbeat with a higher term so that the node reverts to
// a follower state
var leaderId = Guid.NewGuid();
var requesterId = Guid.NewGuid();
_raftNode.Tell(new Request <AppendEntries>(requesterId,
new AppendEntries(42, leaderId, 0, 0, new object[0], 0)));
// Wait for a response
outbox.BlockUntilAny(msg =>
msg is Response <AppendEntries> response && response.ResponseMessage is AppendEntriesResult ae &&
ae.Term == 42);
var result = outbox.CastAs <Response <AppendEntries> >().First();
Assert.IsType <AppendEntriesResult>(result.ResponseMessage);
var responseMessage = (AppendEntriesResult)result.ResponseMessage;
Assert.True(responseMessage.Success);
// There should also be a state change that shows
// that the candidate node reverted back to a follower state
eventLog.ShouldHaveAtLeastOne(msg => msg is RoleStateChanged rsc && rsc.NewState == RoleState.Follower && rsc.Term > 42);
}
public void ShouldAcceptAppendEntriesOnFirstRequest()
{
var numberOfOtherActors = 3;
var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid());
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var startingTerm = 0;
var leaderId = Guid.NewGuid();
var nodeId = Guid.NewGuid();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm);
_raftNode.Tell(new Initialize());
var term = 42;
var response = _raftNode.Request(nodeId,
() => new AppendEntries(term, leaderId, 0, 0, new object[] { "Hello, World" }, 1));
// The response must be valid
Assert.NotEqual(Response <AppendEntries> .Empty, response);
Assert.IsType <AppendEntriesResult>(response.ResponseMessage);
// The first entry must be successful since there are no prior entries
var firstResult = (AppendEntriesResult)response.ResponseMessage;
Assert.True(firstResult.Success);
Assert.Equal(term, firstResult.Term);
}
public void ShouldSendRequestVoteToOtherActorsInTheClusterIfElectionTimerExpires()
{
var minMilliseconds = 150;
var maxMilliseconds = 300;
var nodeId = Guid.NewGuid();
var term = 42;
var numberOfActorsInCluster = 5;
var actorIds = Enumerable.Range(0, numberOfActorsInCluster)
.Select(_ => Guid.NewGuid()).ToArray();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => actorIds, term);
// Set the request timeout to be from 150-300ms
var requesterId = Guid.NewGuid();
_raftNode.Request(requesterId, () => new SetElectionTimeoutRange(minMilliseconds, maxMilliseconds));
// Start the node
_raftNode.Tell(new Initialize());
// Let the timer expire
Thread.Sleep(1000);
var voteRequests = outbox.Where(msg => msg is Request <RequestVote> rv && rv.RequestMessage is RequestVote)
.Cast <Request <RequestVote> >().ToArray();
Assert.NotEmpty(voteRequests);
Assert.True(voteRequests.Count() == numberOfActorsInCluster);
for (var i = 0; i < numberOfActorsInCluster; i++)
{
var request = voteRequests[i];
var voteRequest = request.RequestMessage;
Assert.Equal(nodeId, voteRequest.CandidateId);
// Note: The new candidate must increment the current vote by one
Assert.Equal(term + 1, voteRequest.Term);
}
}
public void ShouldResetElectionTimerWhenVoteIsGrantedToCandidate()
{
// Route all the network output to the collection object
var nodeId = Guid.NewGuid();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]);
// The first response should be DateTime.Never
var requesterId = Guid.NewGuid();
var response = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(response);
Assert.IsType <DateTime>(response.ResponseMessage);
var lastUpdated = (DateTime)response.ResponseMessage;
Assert.True(lastUpdated.Equals(default(DateTime)));
Assert.True(lastUpdated.Equals(DateTime.MinValue));
var currentTime = DateTime.UtcNow;
var candidateId = Guid.NewGuid();
var requestVote = new Request <RequestVote>(requesterId, new RequestVote(42, candidateId, 0, 0));
_raftNode.Tell(requestVote);
var secondResponse = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(secondResponse);
Assert.IsType <DateTime>(secondResponse.ResponseMessage);
// The last updated timestamp should be relatively similar to the current time
var timestamp = (DateTime)secondResponse.ResponseMessage;
var timeDifference = timestamp - currentTime;
Assert.True(timeDifference.TotalMilliseconds >= 0 && timeDifference <= TimeSpan.FromSeconds(5));
}
public void ShouldRejectAppendEntriesIfFollowerCannotFindAMatchForAnEntryInItsOwnLog()
{
/* When sending an AppendEntries RPC,
* the leader includes the term number and index of the entry
* that immediately precedes the new entry.
*
* If the follower cannot find a match for this entry in its own log,
* it rejects the request to append the new entry.
*/
var nodeId = Guid.NewGuid();
var numberOfOtherActors = 3;
var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid());
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var startingTerm = 0;
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm);
_raftNode.Tell(new Initialize());
Thread.Sleep(100);
var term = 42;
var leaderId = Guid.NewGuid();
var appendEntries = new AppendEntries(term, leaderId, 1, 41, new object[0], 0);
var requesterId = Guid.NewGuid();
var response = _raftNode.Request(requesterId, () => appendEntries);
Assert.Equal(requesterId, response.RequesterId);
Assert.Equal(nodeId, response.ResponderId);
// Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm (§5.3)
Assert.IsType <AppendEntriesResult>(response.ResponseMessage);
var result = (AppendEntriesResult)response.ResponseMessage;
Assert.False(result.Success);
}
public void ShouldEmitChangeEventsWhenChangingRoles()
{
var nodeId = Guid.NewGuid();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]);
// Start the node and let it time out
_raftNode.Tell(new Initialize());
Thread.Sleep(500);
bool ShouldContainChangeEvent(object msg)
{
return(msg is RoleStateChanged rsc &&
rsc.ActorId == nodeId &&
rsc.OldState == RoleState.Follower &&
rsc.NewState == RoleState.Candidate);
}
Assert.NotEmpty(eventLog);
Assert.True(eventLog.Count(ShouldContainChangeEvent) > 0);
}
public void ShouldStartNewElectionIfElectionTimeoutOccurs()
{
var numberOfOtherActors = 3;
var actorIds = Enumerable.Range(0, numberOfOtherActors)
.Select(_ => Guid.NewGuid()).ToArray();
Func <IEnumerable <Guid> > getOtherActors = () => actorIds;
var electionTimeoutInMilliseconds = 1000;
var initialTerm = 0;
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var nodeId = Guid.NewGuid();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors, initialTerm,
electionTimeoutInMilliseconds);
// Start the node and let it timeout to trigger an election
_raftNode.Tell(new Initialize());
Thread.Sleep(200);
// Verify that the vote requests have been sent
Assert.True(
outbox.Count(msg => msg is Request <RequestVote> rv && rv.RequestMessage.Term == initialTerm + 1) ==
numberOfOtherActors);
// Avoid sending any votes so that the election has to restart
Thread.Sleep(electionTimeoutInMilliseconds);
// If another round of Request<RequestVote> messages goes out, it means
// that the node has restarted the election
Assert.True(
outbox.Count(msg => msg is Request <RequestVote> rv && rv.RequestMessage.Term == initialTerm + 2) ==
numberOfOtherActors);
}
public void ShouldWinElectionIfMajorityOfVotesReceived()
{
var numberOfOtherActors = 9;
var actorIds = Enumerable.Range(0, numberOfOtherActors)
.Select(_ => Guid.NewGuid()).ToArray();
Func <IEnumerable <Guid> > getOtherActors = () => actorIds;
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var nodeId = Guid.NewGuid();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors);
var numberOfSuccessfulVotes = 5;
var numberOfFailedVotes = 4;
IEnumerable <Response <RequestVote> > CreateVotes(int term, bool result, int numberOfVotes) =>
Enumerable.Range(0, numberOfVotes).Select(index => new Response <RequestVote>(nodeId,
actorIds[index], new RequestVoteResult(term, result, actorIds[index], nodeId)));
// Create an election where 5 out of 9 votes are in favor of the
// candidate node
var newTerm = 1;
var successfulVotes = CreateVotes(newTerm, true, numberOfSuccessfulVotes);
var failedVotes = CreateVotes(newTerm, false, numberOfFailedVotes);
var combinedVotes = successfulVotes
.Union(failedVotes).ToArray();
// Start the node and let the election timeout expire
// in order to trigger a new election
_raftNode.Tell(new Initialize());
Thread.Sleep(200);
foreach (var actorId in getOtherActors())
{
// Verify the contents of every vote request sent out
// by the node
bool ShouldContainVoteRequest(object msg)
{
if (msg is Request <RequestVote> rrv &&
rrv.RequestMessage is RequestVote requestVote)
{
return(rrv.RequesterId == actorId &&
requestVote.CandidateId == nodeId &&
requestVote.Term == newTerm);
}
return(false);
}
Assert.True(outbox.Count(ShouldContainVoteRequest) > 0);
}
// Send the vote responses back to the node
var source = new CancellationTokenSource();
var token = source.Token;
var tasks = combinedVotes.Select(vote => _raftNode.TellAsync(new Context(vote, outbox.Add, token)))
.ToArray();
// Wait until all vote responses have been sent back to the node
Task.WaitAll(tasks);
// The node should post an election outcome message
Assert.NotEmpty(eventLog);
var outcome = eventLog.Where(msg => msg != null && msg is ElectionOutcome)
.Cast <ElectionOutcome>()
.First();
Assert.Equal(nodeId, outcome.WinningActorId);
Assert.Equal(newTerm, outcome.Term);
Assert.Subset(actorIds.ToHashSet(), outcome.KnownActors.ToHashSet());
// Verify the votes
var quorumCount = actorIds.Length * .51;
var matchingVotes = 0;
foreach (var vote in combinedVotes)
{
Assert.IsType <RequestVoteResult>(vote.ResponseMessage);
var currentVote = (RequestVoteResult)vote.ResponseMessage;
bool HasMatchingVote(RequestVoteResult result)
{
return(currentVote.VoteGranted == result.VoteGranted &&
currentVote.CandidateId == result.CandidateId &&
currentVote.Term == result.Term &&
currentVote.VoterId == result.VoterId);
}
matchingVotes += outcome.Votes.Count(HasMatchingVote);
}
// There should be a majority vote in favor of the candidate
Assert.True(matchingVotes >= quorumCount);
}
