public void ShouldBeAbleToGetLastTimestampOfLatestHeartbeat() { // The first response should be DateTime.Never var requesterId = Guid.NewGuid(); var response = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp()); Assert.NotNull(response); Assert.IsType <DateTime>(response.ResponseMessage); var lastUpdated = (DateTime)response.ResponseMessage; Assert.True(lastUpdated.Equals(default(DateTime))); Assert.True(lastUpdated.Equals(DateTime.MinValue)); var currentTime = DateTime.UtcNow; var appendEntries = new Request <AppendEntries>(requesterId, new AppendEntries(0, Guid.NewGuid(), 0, 0, new object[0], 0)); _raftNode.Tell(appendEntries); var secondResponse = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp()); Assert.NotNull(secondResponse); Assert.IsType <DateTime>(secondResponse.ResponseMessage); // The last updated timestamp should be relatively similar to the current time var timestamp = (DateTime)secondResponse.ResponseMessage; var timeDifference = timestamp - currentTime; Assert.True(timeDifference.TotalMilliseconds >= 0 && timeDifference <= TimeSpan.FromSeconds(5)); }
public void ShouldBecomeFollowerIfAppendEntriesReceivedFromNewLeader() { var electionTimeoutInMilliseconds = 1000; var initialTerm = 0; var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); var nodeId = Guid.NewGuid(); var numberOfOtherActors = 2; IEnumerable <Guid> getOtherActors() { return(Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid())); } _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors, initialTerm, electionTimeoutInMilliseconds); void BlockUntilTrue(Func <bool> condition) { while (!condition()) { } } // Start the node and let it timeout to trigger an election _raftNode.Tell(new Initialize()); eventLog.BlockUntilAny(msg => msg is RoleStateChanged rsc && rsc.Term == initialTerm); // Look for the change in state to a candidate node Assert.True(eventLog.Count(msg => msg is RoleStateChanged rsc && rsc.NewState == RoleState.Candidate && rsc.Term == initialTerm) == 1); // Send a single heartbeat with a higher term so that the node reverts to // a follower state var leaderId = Guid.NewGuid(); var requesterId = Guid.NewGuid(); _raftNode.Tell(new Request <AppendEntries>(requesterId, new AppendEntries(42, leaderId, 0, 0, new object[0], 0))); // Wait for a response outbox.BlockUntilAny(msg => msg is Response <AppendEntries> response && response.ResponseMessage is AppendEntriesResult ae && ae.Term == 42); var result = outbox.CastAs <Response <AppendEntries> >().First(); Assert.IsType <AppendEntriesResult>(result.ResponseMessage); var responseMessage = (AppendEntriesResult)result.ResponseMessage; Assert.True(responseMessage.Success); // There should also be a state change that shows // that the candidate node reverted back to a follower state eventLog.ShouldHaveAtLeastOne(msg => msg is RoleStateChanged rsc && rsc.NewState == RoleState.Follower && rsc.Term > 42); }
public void ShouldAcceptAppendEntriesOnFirstRequest() { var numberOfOtherActors = 3; var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid()); var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); var startingTerm = 0; var leaderId = Guid.NewGuid(); var nodeId = Guid.NewGuid(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm); _raftNode.Tell(new Initialize()); var term = 42; var response = _raftNode.Request(nodeId, () => new AppendEntries(term, leaderId, 0, 0, new object[] { "Hello, World" }, 1)); // The response must be valid Assert.NotEqual(Response <AppendEntries> .Empty, response); Assert.IsType <AppendEntriesResult>(response.ResponseMessage); // The first entry must be successful since there are no prior entries var firstResult = (AppendEntriesResult)response.ResponseMessage; Assert.True(firstResult.Success); Assert.Equal(term, firstResult.Term); }
public void ShouldSendRequestVoteToOtherActorsInTheClusterIfElectionTimerExpires() { var minMilliseconds = 150; var maxMilliseconds = 300; var nodeId = Guid.NewGuid(); var term = 42; var numberOfActorsInCluster = 5; var actorIds = Enumerable.Range(0, numberOfActorsInCluster) .Select(_ => Guid.NewGuid()).ToArray(); var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => actorIds, term); // Set the request timeout to be from 150-300ms var requesterId = Guid.NewGuid(); _raftNode.Request(requesterId, () => new SetElectionTimeoutRange(minMilliseconds, maxMilliseconds)); // Start the node _raftNode.Tell(new Initialize()); // Let the timer expire Thread.Sleep(1000); var voteRequests = outbox.Where(msg => msg is Request <RequestVote> rv && rv.RequestMessage is RequestVote) .Cast <Request <RequestVote> >().ToArray(); Assert.NotEmpty(voteRequests); Assert.True(voteRequests.Count() == numberOfActorsInCluster); for (var i = 0; i < numberOfActorsInCluster; i++) { var request = voteRequests[i]; var voteRequest = request.RequestMessage; Assert.Equal(nodeId, voteRequest.CandidateId); // Note: The new candidate must increment the current vote by one Assert.Equal(term + 1, voteRequest.Term); } }
public void ShouldResetElectionTimerWhenVoteIsGrantedToCandidate() { // Route all the network output to the collection object var nodeId = Guid.NewGuid(); var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]); // The first response should be DateTime.Never var requesterId = Guid.NewGuid(); var response = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp()); Assert.NotNull(response); Assert.IsType <DateTime>(response.ResponseMessage); var lastUpdated = (DateTime)response.ResponseMessage; Assert.True(lastUpdated.Equals(default(DateTime))); Assert.True(lastUpdated.Equals(DateTime.MinValue)); var currentTime = DateTime.UtcNow; var candidateId = Guid.NewGuid(); var requestVote = new Request <RequestVote>(requesterId, new RequestVote(42, candidateId, 0, 0)); _raftNode.Tell(requestVote); var secondResponse = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp()); Assert.NotNull(secondResponse); Assert.IsType <DateTime>(secondResponse.ResponseMessage); // The last updated timestamp should be relatively similar to the current time var timestamp = (DateTime)secondResponse.ResponseMessage; var timeDifference = timestamp - currentTime; Assert.True(timeDifference.TotalMilliseconds >= 0 && timeDifference <= TimeSpan.FromSeconds(5)); }
public void ShouldRejectAppendEntriesIfFollowerCannotFindAMatchForAnEntryInItsOwnLog() { /* When sending an AppendEntries RPC, * the leader includes the term number and index of the entry * that immediately precedes the new entry. * * If the follower cannot find a match for this entry in its own log, * it rejects the request to append the new entry. */ var nodeId = Guid.NewGuid(); var numberOfOtherActors = 3; var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid()); var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); var startingTerm = 0; _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm); _raftNode.Tell(new Initialize()); Thread.Sleep(100); var term = 42; var leaderId = Guid.NewGuid(); var appendEntries = new AppendEntries(term, leaderId, 1, 41, new object[0], 0); var requesterId = Guid.NewGuid(); var response = _raftNode.Request(requesterId, () => appendEntries); Assert.Equal(requesterId, response.RequesterId); Assert.Equal(nodeId, response.ResponderId); // Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm (§5.3) Assert.IsType <AppendEntriesResult>(response.ResponseMessage); var result = (AppendEntriesResult)response.ResponseMessage; Assert.False(result.Success); }
public void ShouldEmitChangeEventsWhenChangingRoles() { var nodeId = Guid.NewGuid(); var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]); // Start the node and let it time out _raftNode.Tell(new Initialize()); Thread.Sleep(500); bool ShouldContainChangeEvent(object msg) { return(msg is RoleStateChanged rsc && rsc.ActorId == nodeId && rsc.OldState == RoleState.Follower && rsc.NewState == RoleState.Candidate); } Assert.NotEmpty(eventLog); Assert.True(eventLog.Count(ShouldContainChangeEvent) > 0); }
public void ShouldStartNewElectionIfElectionTimeoutOccurs() { var numberOfOtherActors = 3; var actorIds = Enumerable.Range(0, numberOfOtherActors) .Select(_ => Guid.NewGuid()).ToArray(); Func <IEnumerable <Guid> > getOtherActors = () => actorIds; var electionTimeoutInMilliseconds = 1000; var initialTerm = 0; var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); var nodeId = Guid.NewGuid(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors, initialTerm, electionTimeoutInMilliseconds); // Start the node and let it timeout to trigger an election _raftNode.Tell(new Initialize()); Thread.Sleep(200); // Verify that the vote requests have been sent Assert.True( outbox.Count(msg => msg is Request <RequestVote> rv && rv.RequestMessage.Term == initialTerm + 1) == numberOfOtherActors); // Avoid sending any votes so that the election has to restart Thread.Sleep(electionTimeoutInMilliseconds); // If another round of Request<RequestVote> messages goes out, it means // that the node has restarted the election Assert.True( outbox.Count(msg => msg is Request <RequestVote> rv && rv.RequestMessage.Term == initialTerm + 2) == numberOfOtherActors); }
public void ShouldWinElectionIfMajorityOfVotesReceived() { var numberOfOtherActors = 9; var actorIds = Enumerable.Range(0, numberOfOtherActors) .Select(_ => Guid.NewGuid()).ToArray(); Func <IEnumerable <Guid> > getOtherActors = () => actorIds; var outbox = new ConcurrentBag <object>(); var eventLog = new ConcurrentBag <object>(); var nodeId = Guid.NewGuid(); _raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors); var numberOfSuccessfulVotes = 5; var numberOfFailedVotes = 4; IEnumerable <Response <RequestVote> > CreateVotes(int term, bool result, int numberOfVotes) => Enumerable.Range(0, numberOfVotes).Select(index => new Response <RequestVote>(nodeId, actorIds[index], new RequestVoteResult(term, result, actorIds[index], nodeId))); // Create an election where 5 out of 9 votes are in favor of the // candidate node var newTerm = 1; var successfulVotes = CreateVotes(newTerm, true, numberOfSuccessfulVotes); var failedVotes = CreateVotes(newTerm, false, numberOfFailedVotes); var combinedVotes = successfulVotes .Union(failedVotes).ToArray(); // Start the node and let the election timeout expire // in order to trigger a new election _raftNode.Tell(new Initialize()); Thread.Sleep(200); foreach (var actorId in getOtherActors()) { // Verify the contents of every vote request sent out // by the node bool ShouldContainVoteRequest(object msg) { if (msg is Request <RequestVote> rrv && rrv.RequestMessage is RequestVote requestVote) { return(rrv.RequesterId == actorId && requestVote.CandidateId == nodeId && requestVote.Term == newTerm); } return(false); } Assert.True(outbox.Count(ShouldContainVoteRequest) > 0); } // Send the vote responses back to the node var source = new CancellationTokenSource(); var token = source.Token; var tasks = combinedVotes.Select(vote => _raftNode.TellAsync(new Context(vote, outbox.Add, token))) .ToArray(); // Wait until all vote responses have been sent back to the node Task.WaitAll(tasks); // The node should post an election outcome message Assert.NotEmpty(eventLog); var outcome = eventLog.Where(msg => msg != null && msg is ElectionOutcome) .Cast <ElectionOutcome>() .First(); Assert.Equal(nodeId, outcome.WinningActorId); Assert.Equal(newTerm, outcome.Term); Assert.Subset(actorIds.ToHashSet(), outcome.KnownActors.ToHashSet()); // Verify the votes var quorumCount = actorIds.Length * .51; var matchingVotes = 0; foreach (var vote in combinedVotes) { Assert.IsType <RequestVoteResult>(vote.ResponseMessage); var currentVote = (RequestVoteResult)vote.ResponseMessage; bool HasMatchingVote(RequestVoteResult result) { return(currentVote.VoteGranted == result.VoteGranted && currentVote.CandidateId == result.CandidateId && currentVote.Term == result.Term && currentVote.VoterId == result.VoterId); } matchingVotes += outcome.Votes.Count(HasMatchingVote); } // There should be a majority vote in favor of the candidate Assert.True(matchingVotes >= quorumCount); }