public async Task ShouldAppendCommandToLocalLog()
{
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
var peer = new RemoteControledPeer();
peer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
_peers.Add(peer);
}
var log = new InMemoryLog();
_currentState = new CurrentState(_id, 0, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, log, _node, _settings, _rules, _loggerFactory.Object);
await leader.Accept(new FakeCommand());
log.ExposedForTesting.Count.ShouldBe(1);
bool PeersReceiveCorrectAppendEntries(List <IPeer> peers)
{
var passed = 0;
peers.ForEach(p =>
{
var rc = (RemoteControledPeer)p;
if (rc.AppendEntriesResponsesWithLogEntries == 1)
{
passed++;
}
});
return(passed == peers.Count);
}
var result = WaitFor(1000).Until(() => PeersReceiveCorrectAppendEntries(_peers));
result.ShouldBeTrue();
bool FirstTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 1)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 2)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
result = WaitFor(1000).Until(() => FirstTest(leader.PeerStates));
result.ShouldBeTrue();
}
public async Task ShouldSetCommitIndex()
{
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
var peer = new RemoteControledPeer();
peer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
_peers.Add(peer);
}
//add 3 logs
_currentState = new CurrentState(_id, 1, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, _log, _node, _settings, _rules, _loggerFactory.Object);
await leader.Accept(new FakeCommand());
await leader.Accept(new FakeCommand());
await leader.Accept(new FakeCommand());
bool PeersTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 3)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 4)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
var result = WaitFor(2000).Until(() => PeersTest(leader.PeerStates));
leader.CurrentState.CommitIndex.ShouldBe(3);
result.ShouldBeTrue();
}
public async Task ShouldHandleCommandIfNoPeers()
{
_peers = new List <IPeer>();
var log = new InMemoryLog();
_currentState = new CurrentState(_id, 0, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, log, _node, _settings, _rules, _loggerFactory.Object);
var response = await leader.Accept <FakeCommand>(new FakeCommand());
log.ExposedForTesting.Count.ShouldBe(1);
var fsm = (InMemoryStateMachine)_fsm;
fsm.HandledLogEntries.ShouldBe(1);
response.ShouldBeOfType <OkResponse <FakeCommand> >();
}
public void ShouldHandleCommandIfNoPeers()
{
_peers = new List <IPeer>();
var log = new InMemoryLog();
_currentState = new CurrentState(_id, 0, default(Guid), 0, 0, default(Guid));
var leader = new Leader(_currentState, _fsm, (s) => _peers, log, _node, _settings, _rules);
var response = leader.Accept <FakeCommand>(new FakeCommand());
log.ExposedForTesting.Count.ShouldBe(1);
var fsm = (InMemoryStateMachine)_fsm;
fsm.ExposedForTesting.ShouldBe(1);
response.ShouldBeOfType <OkResponse <FakeCommand> >();
}
public async Task ShouldAppendCommandToLocalLog()
{
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
var peer = new RemoteControledPeer();
peer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
_peers.Add(peer);
}
var log = new InMemoryLog();
_currentState = new CurrentState(_id, 0, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, log, _node, _settings, _rules);
await leader.Accept(new FakeCommand());
log.ExposedForTesting.Count.ShouldBe(1);
}
public async Task ShouldTimeoutAfterXSecondsIfCannotReplicateCommandAndRollbackIndexes()
{
_peers = new List <IPeer>();
for (var i = 0; i < 3; i++)
{
_peers.Add(new FakePeer(false, false, false));
}
_peers.Add(new FakePeer(true, true, true));
_currentState = new CurrentState(_id, 1, default(string), 0, 0, default(string));
_settings = new InMemorySettingsBuilder().WithCommandTimeout(1).Build();
var leader = new Leader(_currentState, _fsm, (s) => _peers, _log, _node, _settings, _rules, _loggerFactory.Object);
var command = new FakeCommand();
var response = await leader.Accept(command);
var error = (ErrorResponse <FakeCommand>)response;
error.Error.ShouldBe("Unable to replicate command to peers due to timeout.");
bool TestPeerStates(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 0)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 1)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
var result = WaitFor(1000).Until(() => TestPeerStates(leader.PeerStates));
_log.Count().Result.ShouldBe(0);
result.ShouldBeTrue();
}
public async Task ShouldApplyCommandToStateMachine()
{
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
var peer = new RemoteControledPeer();
peer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
_peers.Add(peer);
}
var log = new InMemoryLog();
_currentState = new CurrentState(_id, 0, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, log, _node, _settings, _rules, _loggerFactory.Object);
var response = await leader.Accept <FakeCommand>(new FakeCommand());
log.ExposedForTesting.Count.ShouldBe(1);
var fsm = (InMemoryStateMachine)_fsm;
fsm.HandledLogEntries.ShouldBe(1);
response.ShouldBeOfType <OkResponse <FakeCommand> >();
}
public async Task ShouldReplicateCommand()
{
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
_peers.Add(new FakePeer(true, true, true));
}
_currentState = new CurrentState(_id, 1, default(string), 0, 0, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, _log, _node, _settings, _rules, _loggerFactory.Object);
var command = new FakeCommand();
var response = await leader.Accept(command);
response.ShouldBeOfType <OkResponse <FakeCommand> >();
bool TestPeerStates(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 1)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 2)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
var result = WaitFor(1000).Until(() => TestPeerStates(leader.PeerStates));
result.ShouldBeTrue();
}
public async Task ShouldDecrementNextIndexAndRetry()
{
//create peers that will initially return false when asked to append entries...
_peers = new List <IPeer>();
for (var i = 0; i < 4; i++)
{
var peer = new RemoteControledPeer();
peer.SetAppendEntriesResponse(new AppendEntriesResponse(1, false));
_peers.Add(peer);
}
_currentState = new CurrentState(_id, 1, default(string), 1, 1, default(string));
var leader = new Leader(_currentState, _fsm, (s) => _peers, _log, _node, _settings, _rules, _loggerFactory.Object);
//send first command, this wont get commited because the guys are replying false
var task = Task.Run(async() => await leader.Accept(new FakeCommand()));
bool FirstTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 0)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 1)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
var result = WaitFor(1000).Until(() => FirstTest(leader.PeerStates));
result.ShouldBeTrue();
//now the peers accept the append entries
foreach (var peer in _peers)
{
var rcPeer = (RemoteControledPeer)peer;
rcPeer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
}
//wait on sending the command
task.Wait();
bool SecondTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 1)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 2)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
result = WaitFor(1000).Until(() => SecondTest(leader.PeerStates));
result.ShouldBeTrue();
//now the peers stop accepting append entries..
foreach (var peer in _peers)
{
var rcPeer = (RemoteControledPeer)peer;
rcPeer.SetAppendEntriesResponse(new AppendEntriesResponse(1, false));
}
//send another command, this wont get commited because the guys are replying false
task = Task.Run(async() => await leader.Accept(new FakeCommand()));
bool ThirdTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 1)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 2)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
result = WaitFor(1000).Until(() => ThirdTest(leader.PeerStates));
result.ShouldBeTrue();
//now the peers accept the append entries
foreach (var peer in _peers)
{
var rcPeer = (RemoteControledPeer)peer;
rcPeer.SetAppendEntriesResponse(new AppendEntriesResponse(1, true));
}
task.Wait();
bool FourthTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 2)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 3)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
result = WaitFor(1000).Until(() => FourthTest(leader.PeerStates));
result.ShouldBeTrue();
//send another command
await leader.Accept(new FakeCommand());
bool FirthTest(List <PeerState> peerState)
{
var passed = 0;
peerState.ForEach(pS =>
{
if (pS.MatchIndex.IndexOfHighestKnownReplicatedLog == 3)
{
passed++;
}
if (pS.NextIndex.NextLogIndexToSendToPeer == 4)
{
passed++;
}
});
return(passed == peerState.Count * 2);
}
result = WaitFor(2000).Until(() => FirthTest(leader.PeerStates));
result.ShouldBeTrue();
}