public void LeadsFollowersAndTheirHeartbeatLikeALeader()
{
var list = new ConcurrentBag <FriendlyPeer>();
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(
() => {
var s = new FriendlyPeer();
list.Add(s);
return(s);
}
);
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysOldTimestamp();
_server.Start();
Thread.Sleep(3000);
Assert.Equal(Role.Leader, _server.Role);
var timesFollowersServed = list.SelectMany(x => x.AllThemAppendEntriesRequests.Where(y => y.Entries.Length == 0)).Count();
Assert.True(timesFollowersServed > 10);
_output.WriteLine($"{nameof(timesFollowersServed)}: {timesFollowersServed}");
}
public async Task CreatesSnapshotAsALeader()
{
_settings.MinSnapshottingIndexInterval = 10L;
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(_mockPeer.Object);
_mockPeer.Setup(x => x.RequestVoteAsync(It.IsAny <RequestVoteRequest>())).ReturnsAsync(
new RequestVoteResponse()
{
CurrentTerm = 0,
VoteGranted = true
}
);
_mockPeer.Setup(x => x.AppendEntriesAsync(It.IsAny <AppendEntriesRequest>())).ReturnsAsync(
new AppendEntriesResponse(1, true)
);
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysOldTimestamp();
_server.Start();
Thread.Sleep(1000); // must be a leader by now
Assert.Equal(Role.Leader, _server.Role);
for (var i = 0; i < 15; i++)
{
await _server.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = new byte[128]
});
}
Thread.Sleep(3000);
_mockPeer.VerifyAll();
}
public async Task RunsErrandsForLogs()
{
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
// has to be done - otherwise it becomes a candidate
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
var currentPosition = 0;
var term = 2;
for (int i = 0; i < 10; i++)
{
var entries = GetSomeRandomEntries();
await _server.AppendEntriesAsync(new AppendEntriesRequest()
{
CurrentTerm = term,
Entries = entries,
LeaderCommitIndex = currentPosition + entries.Length,
LeaderId = leaderId,
PreviousLogIndex = currentPosition - 1,
PreviousLogTerm = term
});
currentPosition += entries.Length;
}
Assert.Equal(currentPosition - 1, _sister.LastIndex);
Assert.Equal(Role.Follower, _server.Role);
}
public async Task NeverVotesTwice()
{
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
_server.Start();
Thread.Sleep(400);
var result = await _server.RequestVoteAsync(new RequestVoteRequest()
{
CandidateId = Guid.NewGuid(),
CurrentTerm = 2,
LastLogIndex = 2,
LastLogTerm = 1
});
Thread.Sleep(400);
var result2 = await _server.RequestVoteAsync(new RequestVoteRequest()
{
CandidateId = Guid.NewGuid(),
CurrentTerm = 2,
LastLogIndex = 2,
LastLogTerm = 1
});
Assert.False(result2.VoteGranted);
}
public async Task NeverVotesYesToLowerTerm()
{
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
// to set the term to 1
var result = await _server.RequestVoteAsync(new RequestVoteRequest()
{
CandidateId = Guid.NewGuid(),
CurrentTerm = 1,
LastLogIndex = 2,
LastLogTerm = 1
});
Thread.Sleep(400); // now it becomes candidate and returns no
// now its term already moved to 2 due to timeout
result = await _server.RequestVoteAsync(new RequestVoteRequest()
{
CandidateId = Guid.NewGuid(),
CurrentTerm = 1,
LastLogIndex = 2,
LastLogTerm = 1
});
Assert.False(result.VoteGranted);
Assert.Equal(2, result.CurrentTerm);
}
public async Task KeepFeedingAFollowerAndNeverDreamsOfPower()
{
var t = new CancellationTokenSource();
var leaderId = Guid.NewGuid();
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
// to set the term to 1
await _server.AppendEntriesAsync(new AppendEntriesRequest()
{
CurrentTerm = 1,
Entries = new byte[0][],
LeaderCommitIndex = 20,
LeaderId = leaderId,
PreviousLogIndex = -1,
PreviousLogTerm = 0
});
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
TheTrace.TraceInformation("OK, now this is before wait...");
Thread.Sleep(1000);
TheTrace.TraceInformation("Wait finished.");
Assert.Equal(1, _server.State.CurrentTerm);
TheTrace.TraceInformation("Checked Term.");
Assert.Equal(Role.Follower, _server.Role);
TheTrace.TraceInformation("Checked Role.");
t.Cancel();
}
public void PerpetualCandidacyWhenDealingWithLazyPeers()
{
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(new LazyPeer());
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
Thread.Sleep(2000);
Assert.Equal(Role.Candidate, _server.Role);
}
public void FriendlyPeersWithOneAngryNotABigDeal()
{
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(new FriendlyPeer());
_manijer.Setup(x => x.GetProxy(It.Is <string>(y => y == "7"))).Returns(new AngryPeer());
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
Thread.Sleep(1000);
Assert.Equal(Role.Leader, _server.Role);
}
public void BackStabberPeersMakeYouFollower()
{
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(new BackStabberPeer());
_manijer.Setup(x => x.GetProxy(It.Is <string>(y => y == "7"))).Returns(new FriendlyPeer());
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.Start();
TheTrace.TraceInformation("OK, now this is before wait...");
Thread.Sleep(300);
TheTrace.TraceInformation("Wait finished.");
Assert.True(1 >= _server.State.CurrentTerm);
TheTrace.TraceInformation("Checked Term.");
Assert.True(Role.Follower == _server.Role || Role.Candidate == _server.Role);
TheTrace.TraceInformation("Checked Role.");
}
public async Task LeadsFollowersAndTheirLogsLikeALeader()
{
var list = new ConcurrentBag <FriendlyPeer>();
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.IsAny <string>())).Returns(
() => {
var s = new FriendlyPeer();
list.Add(s);
return(s);
}
);
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysOldTimestamp();
_server.Start();
Thread.Sleep(2000); // must be a leader by now
Assert.Equal(Role.Leader, _server.Role);
await _server.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = new byte[128]
});
await _server.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = new byte[128]
});
await _server.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = new byte[128]
});
Thread.Sleep(2000);
var timesFollowersServed = list.SelectMany(x => {
var copy = x.AllThemAppendEntriesRequests.ToArray();
return(copy.Where(y => y.Entries.Length > 0));
}).Count();
_output.WriteLine($"timesFollowersServed: {timesFollowersServed}");
Assert.True(timesFollowersServed > 1);
Assert.Equal(2, _sister.LastIndex);
}
private void SetupNodes()
{
foreach (var address in _addresses)
{
TheTrace.TraceInformation($"Setting up {address} ({_peers[address].ShortName}) {_peers[address].Id}");
var lp = new LmdbPersister(_folders[address], seedId: _peers[address].Id)
{
Name = _peers[address].ShortName
};
var peers = _peers.Where(x => x.Key != address).Select(x => x.Value).ToArray();
foreach (var p in peers)
{
TheTrace.TraceInformation($"{p.Address} - ({p.ShortName}) {p.Id}");
}
TheTrace.TraceInformation("___________________________________________________________________________");
var peerManager = new PeerManager(peers, _nodes);
var node = new DefaultRaftServer(lp, lp, lp,
new SimpleDictionaryStateMachine(), peerManager, _settings, _peers[address], chaos:
new SimpleChaos(TimeSpan.FromMilliseconds(200), exceptionProbability: 0.0));
_nodes.Add(address, node);
}
}
public async Task CreatesSnapshotAsAFollower()
{
var leaderId = Guid.NewGuid();
_settings.MinSnapshottingIndexInterval = 10L;
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysRecentTimestamp();
_server.LastHeartBeatSent = new AlwaysRecentTimestamp();
_server.Start();
var term = 1;
var count = 15L;
var entry = new byte[122];
var entries = new [] { entry };
for (int i = 0; i < count; i++)
{
await _server.AppendEntriesAsync(new AppendEntriesRequest()
{
CurrentTerm = term,
Entries = entries,
LeaderCommitIndex = i - 1,
LeaderId = leaderId,
PreviousLogIndex = i - 1,
PreviousLogTerm = term
});
}
Thread.Sleep(1000); // must be a leader by now
Assert.Equal(Role.Follower, _server.Role);
Snapshot ss;
Assert.True(_server.SnapshotOperator.TryGetLastSnapshot(out ss));
Assert.Equal(count - 2, ss.LastIncludedIndex); // last index is 14 and PreviousLogIndex is set to i-1 => 13
Assert.Equal(term, ss.LastIncludedTerm);
Assert.Equal(count - 1, _server.LogPersister.LogOffset);
}
public async Task LeaderSendsSnapshotForAClientSoBehind()
{
_settings.MinSnapshottingIndexInterval = 10L;
_maqina.Setup(x => x.WriteSnapshotAsync(It.IsAny <Stream>()))
.Callback(async(Stream s) => s.Write(new byte[256], 0, 256)).Returns(Task.CompletedTask);
var term = 1L;
_manijer.Setup(x => x.GetPeers()).Returns(new[] { "1", "3", "5", "7" }.Select(s => new Peer(s, Guid.NewGuid())));
_manijer.Setup(x => x.GetProxy(It.Is <string>(s => s != "1"))).Returns(_mockPeer.Object);
var behindPeer = new Mock <IRaftServer>();
_manijer.Setup(x => x.GetProxy(It.Is <string>(s => s == "1"))).Returns(behindPeer.Object);
_mockPeer.Setup(x => x.RequestVoteAsync(It.IsAny <RequestVoteRequest>())).ReturnsAsync(
new RequestVoteResponse()
{
CurrentTerm = term,
VoteGranted = true
}
);
behindPeer.Setup(x => x.RequestVoteAsync(It.IsAny <RequestVoteRequest>())).ReturnsAsync(
new RequestVoteResponse()
{
CurrentTerm = term,
VoteGranted = true
}
);
_mockPeer.Setup(x => x.AppendEntriesAsync(It.IsAny <AppendEntriesRequest>())).ReturnsAsync(
new AppendEntriesResponse(term, true)
);
// unsuccessful response to append entry so that it is left behind
behindPeer.Setup(x => x.AppendEntriesAsync(It.Is <AppendEntriesRequest>(
req => req.Entries.Length > 0
))).ReturnsAsync(
new AppendEntriesResponse(term, false)
);
// but success to heartbeat
behindPeer.Setup(x => x.AppendEntriesAsync(It.Is <AppendEntriesRequest>(
req => req.Entries.Length == 0
))).ReturnsAsync(
new AppendEntriesResponse(term, true)
);
// should be called for installing snapshot
behindPeer.Setup(x => x.InstallSnapshotAsync(It.Is <InstallSnapshotRequest>(req =>
req.CurrentTerm == term && req.LastIncludedIndex == 14
))).ReturnsAsync(
new InstallSnapshotResponse()
{
CurrentTerm = term
}
);
_server = new DefaultRaftServer(_sister, _sister, _sister, _maqina.Object, _manijer.Object, _settings);
_server.LastHeartBeat = new AlwaysOldTimestamp();
_server.Start();
Thread.Sleep(1000); // must be a leader by now
Assert.Equal(Role.Leader, _server.Role);
for (var i = 0; i < 15; i++)
{
await _server.ApplyCommandAsync(new StateMachineCommandRequest()
{
Command = new byte[128]
});
}
Thread.Sleep(3000);
_mockPeer.VerifyAll();
behindPeer.VerifyAll();
}