public void AddServer_ReplyNotLeaderIfNotLeader()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize();
s2.Initialize();
//s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
//s1.Advance();
var testState = new TestState(s2);
s2.ChangeState(testState);
var request = new AddServerRequest()
{
From = s2.ID
};
s2.Transport.SendMessage(new Client(s2, s1.ID), request);
s1.Advance();
s2.Advance();
//s2.Transport.SendMessage(new Client())
Assert.AreEqual(typeof(AddServerReply), testState.LastMessage.GetType());
Assert.AreEqual(AddServerStatus.NotLeader, ((AddServerReply)testState.LastMessage).Status);
}
}
public void LogCommitIndex()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 1;
s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
s2.Advance();
s1.Advance();
s1.PersistedStore.CreateData(s1, new byte[] { 5 });
s1.Advance();
s2.Advance();
//log commit index check
s1.Advance(50);
s2.Advance();
Assert.AreEqual(1u, s1.CommitIndex);
Assert.AreEqual(1u, s2.CommitIndex);
}
}
public void RemoveServer_OK()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize( s2.ID);
s2.Initialize( s1.ID);
s1.ChangeState(new LeaderState(s1));
s1.Advance();
s2.Advance();
s2.ChangeState(new LeaveState(s2));
var count = 200;
while (count-- > 0)
{
s1.Advance();
s2.Advance();
}
Assert.AreEqual(0, s1.Clients.Count());
Assert.AreEqual(typeof(StoppedState), s2.CurrentState.GetType());
}
}
public void AddServer_ReplicateToLog()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize();
s2.Initialize();
s1.ChangeState(new LeaderState(s1)); // will push s1 to term 2
s1.PersistedStore.AddServer(s1, s1.ID);
// applies its own entry and advances commit
s1.Advance();
// this sends out an add request
s2.ChangeState(new JoinState(s2, new Client(s2, s1.ID)));
var count = 200;
while (count-- > 0)
{
s1.Advance();
s2.Advance();
}
Assert.AreEqual(2, s1.Majority);
Assert.AreEqual(2, s2.Majority);
Assert.IsTrue(s1.ID.Equals(s2.GetClient(s1.ID).ID));
Assert.IsTrue(s2.ID.Equals(s1.GetClient(s2.ID).ID));
Assert.IsTrue(s1.PersistedStore.Clients.Any(x => x.Equals(s2.ID)));
Assert.IsTrue(s2.PersistedStore.Clients.Any(x => x.Equals(s1.ID)));
}
}
public void IsFollower()
{
using (var mock = new T())
using (var server = mock.CreateServer())
{
server.Initialize();
server.Advance(1);
Assert.AreEqual(typeof(FollowerState), server.CurrentState.GetType());
}
}
public void IsCandidate()
{
using (var mock = new T())
using (var server = mock.CreateServer())
{
server.Initialize();
var ticks = server.PersistedStore.ELECTION_TIMEOUT * 2;
while (ticks-- > 0 && server.CurrentState is FollowerState)
server.Advance();
Assert.AreEqual(typeof(CandidateState), server.CurrentState.GetType());
}
}
public void IsLeader()
{
using (var mock = new T())
using (var server = mock.CreateServer())
{
server.Initialize();
var count = server.PersistedStore.ELECTION_TIMEOUT * 2;
while (count-- > 0)
{
server.Advance();
}
Assert.AreEqual(typeof(LeaderState), server.CurrentState.GetType());
}
}
public void LogReplicated()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 1;
s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
s2.Advance(5);
s1.Advance(5);
s1.PersistedStore.CreateData(s1, new byte[] { 5 });
s1.Advance(50);
s2.Advance(5);
LogIndex logIndex;
var index = s2.PersistedStore.GetLastIndex(out logIndex);
//log replication check
Assert.AreNotEqual(0u, index);
Assert.AreEqual(2u, logIndex.Term);
Assert.AreEqual(LogIndexType.DataBlob, logIndex.Type);
Assert.AreEqual(0u, logIndex.ChunkOffset);
Assert.AreEqual(1u, logIndex.ChunkSize);
var data = s2.PersistedStore.GetData(logIndex);
Assert.AreEqual(1, data.Length);
Assert.AreEqual((byte)5, data[0]);
}
}
public void NodeGrantsVote()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.ChangeState(new CandidateState(s1));
var count = s1.PersistedStore.ELECTION_TIMEOUT;
while (count-- > 0)
{
s2.Advance();
s1.Advance();
}
Assert.AreEqual(s1.ID, s2.PersistedStore.VotedFor);
Assert.AreEqual(true, s1.Clients.First().VoteGranted);
}
}
public void NodeGrantsVoteWithLongerLogOlderTerm()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 1;
s1.PersistedStore.CreateData(s1, new byte[] { 0 });
s1.PersistedStore.Term = 2;
s1.PersistedStore.CreateData(s1, new byte[] { 1 });
s2.PersistedStore.CreateData(s2, new byte[] { 0 });
s2.PersistedStore.CreateData(s2, new byte[] { 1 });
s2.PersistedStore.CreateData(s2, new byte[] { 2 });
s2.PersistedStore.CreateData(s2, new byte[] { 3 });
s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
var count = s1.PersistedStore.ELECTION_TIMEOUT;
while (count-- > 0)
{
s2.Advance();
s1.Advance();
}
Assert.AreEqual(s1.ID, s2.PersistedStore.VotedFor);
Assert.AreEqual(true, s1.Clients.First().VoteGranted);
}
}
public void AddServer_StillGrantsVote()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize();
s2.Initialize();
s1.ChangeState(new LeaderState(s1)); // will push s1 to term 2
s1.PersistedStore.AddServer(s1, s1.ID);
// applies its own entry and advances commit
s1.Advance();
// this sends out an add request
s2.ChangeState(new JoinState(s2, new Client(s2, s1.ID)));
// these are needed because the first append entries will fail
// and s2 will return where its nextIndex is
s1.Advance();
s2.Advance();
// reads add request and sends its self as the first entry
s1.Advance();
// s2 now has s1 as an added entry and has applied the index
s2.Advance();
// s1 sees that s2 is up to date and adds log entry for s2 and locks config
s1.Advance();
s2.Advance();
s1.Advance();
s1.ChangeState(new CandidateState(s1));
//var count = 50;
//while (count-- > 0)
{
s2.Advance();
s1.Advance();
}
Assert.AreEqual(s1.ID, s2.PersistedStore.VotedFor);
Assert.AreEqual(true, s1.Clients.First().VoteGranted);
}
}
public void AddServer_Timesout()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize();
s2.Initialize();
s1.ChangeState(new LeaderState(s1)); // will push s1 to term 2
s1.PersistedStore.AddServer(s1, s1.ID);
// applies its own entry and advances commit
s1.Advance();
// this sends out an add request
s2.ChangeState(new JoinState(s2, new Client(s2, s1.ID)));
// reads add request and sends its self as the first entry
s1.Advance();
var testState = new TestState(s2);
s2.ChangeState(testState);
s1.Advance(s1.PersistedStore.ELECTION_TIMEOUT);
s2.Advance();
Assert.AreEqual(typeof(AddServerReply), testState.LastMessage.GetType());
Assert.AreEqual(AddServerStatus.TimedOut, ((AddServerReply)testState.LastMessage).Status);
}
}
public void AddServer_ReplicateToLogWithRollback()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize();
s2.Initialize();
s1.ChangeState(new LeaderState(s1)); // will push s1 to term 2
s1.PersistedStore.AddServer(s1, s1.ID);
// applies its own entry and advances commit
s1.Advance();
s1.PersistedStore.CreateData(s1, new byte[] { 1 });
// this sends out an add request
s2.ChangeState(new JoinState(s2, new Client(s2, s1.ID)));
var count = 50;
while (count-- > 0)
{
s1.Advance();
s2.Advance();
}
s1.PersistedStore.Term++;
s1.PersistedStore.CreateData(s1, new byte[] { 2 });
//s1.PersistedStore.CreateData(s1, new byte[65400 * 3]);
s2.PersistedStore.CreateData(s1, new byte[65400 * 3]);
count = 200;
while (count-- > 0)
{
s1.Advance();
s2.Advance();
}
Console.WriteLine("");
Console.WriteLine("");
Log.DumpLog(s1.PersistedStore);
Console.WriteLine("");
Console.WriteLine("");
Log.DumpLog(s2.PersistedStore);
Assert.AreEqual(2, s1.Majority);
Assert.AreEqual(2, s2.Majority);
Assert.IsTrue(s1.ID.Equals(s2.GetClient(s1.ID).ID));
Assert.IsTrue(s2.ID.Equals(s1.GetClient(s2.ID).ID));
Assert.IsTrue(s1.PersistedStore.Clients.Any(x => x.Equals(s2.ID)));
Assert.IsTrue(s2.PersistedStore.Clients.Any(x => x.Equals(s1.ID)));
Assert.IsTrue(Log.AreEqual(s1.PersistedStore, s2.PersistedStore));
}
}
public void NodeDoesntGrantVoteWithNewerTerm()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 3;
s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
var count = s1.PersistedStore.ELECTION_TIMEOUT;
while (count-- > 0)
{
s2.Advance();
s1.Advance();
}
Assert.AreEqual(null, s2.PersistedStore.VotedFor);
Assert.AreEqual(false, s1.Clients.First().VoteGranted);
}
}
public void TestVerify()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 1;
s1.PersistedStore.CreateData(s1, new byte[] { 0 });
s1.PersistedStore.CreateData(s1, new byte[] { 1 });
s2.PersistedStore.CreateData(s2, new byte[] { 0 });
s2.PersistedStore.CreateData(s2, new byte[] { 1 });
s1.ChangeState(new LeaderState(s1));
s1.Advance();
var verifyState = new VerifyState(0, new Client(s2, s1.ID), s2);
s2.ChangeState(verifyState);
s2.Advance();
var index = 50;
while (index-- > 0)
{
s1.Advance();
s2.Advance();
}
Assert.AreEqual(true, verifyState.IsVerified);
}
}
public void TestChunkedLogs()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.ChangeState(new LeaderState(s1));
//establish leader
s1.Advance();
s2.Advance();
//create entry
var data = new byte[1024 * 1024];
for (var i = 0; i < data.Length; i++)
data[i] = (byte)i;
s1.PersistedStore.CreateData(s1, data);
var count = 200;
while (count-- > 0)
{
s1.Advance();
s2.Advance();
}
var addition = (data.Length % Log.MAX_LOG_ENTRY_SIZE) == 0 ? 0u : 1u;
Assert.AreEqual((uint)(data.Length / Log.MAX_LOG_ENTRY_SIZE) + addition, s1.CommitIndex);
Assert.AreEqual((uint)(data.Length / Log.MAX_LOG_ENTRY_SIZE) + addition, s2.CommitIndex);
Assert.AreEqual(0u, s1.PersistedStore[s1.PersistedStore.Length].Flag3);
Assert.AreEqual((uint)data.Length, s1.PersistedStore[s1.PersistedStore.Length].Flag4);
Assert.AreEqual(0u, s2.PersistedStore[s2.PersistedStore.Length].Flag3);
Assert.AreEqual((uint)data.Length, s2.PersistedStore[s2.PersistedStore.Length].Flag4);
for (var i = 0; i < (uint)(data.Length / Log.MAX_LOG_ENTRY_SIZE) - 1; i++)
{
Assert.AreEqual((uint)(i * Log.MAX_LOG_ENTRY_SIZE), s1.PersistedStore[(uint)i + 1].ChunkOffset);
Assert.AreEqual((uint)Log.MAX_LOG_ENTRY_SIZE, s1.PersistedStore[(uint)i + 1].ChunkSize);
Assert.AreEqual((uint)(i * Log.MAX_LOG_ENTRY_SIZE), s2.PersistedStore[(uint)i + 1].ChunkOffset);
Assert.AreEqual((uint)Log.MAX_LOG_ENTRY_SIZE, s2.PersistedStore[(uint)i + 1].ChunkSize);
Assert.AreEqual(0u, s1.PersistedStore[(uint)i + 1].Flag3);
Assert.AreEqual(0u, s2.PersistedStore[(uint)i + 1].Flag3);
Assert.AreEqual((uint)data.Length, s1.PersistedStore[(uint)i + 1].Flag4);
Assert.AreEqual((uint)data.Length, s2.PersistedStore[(uint)i + 1].Flag4);
}
var logIndex = s2.PersistedStore[s2.PersistedStore.Length];
Assert.AreEqual(1u, logIndex.Term);
var storedData = new byte[logIndex.Flag4];
Assert.AreEqual(data.Length, storedData.Length);
using (var stream = s2.PersistedStore.GetDataStream())
{
stream.Seek(logIndex.Flag3, System.IO.SeekOrigin.Begin);
stream.Read(storedData, 0, storedData.Length);
}
for (var i = 0; i < data.Length; i++)
{
Assert.AreEqual(data[i], storedData[i]);
}
}
}
public void NodeGrantsVoteWithSameLog()
{
using (var mock = new T())
using (var s1 = mock.CreateServer())
using (var s2 = mock.CreateServer())
{
s1.Initialize(s2.ID);
s2.Initialize(s1.ID);
s1.PersistedStore.Term = 1;
s2.PersistedStore.Term = 1;
s1.PersistedStore.CreateData(s1, new byte[] { 0 });
s1.PersistedStore.CreateData(s1, new byte[] { 1 });
s2.PersistedStore.CreateData(s2, new byte[] { 0 });
s2.PersistedStore.CreateData(s2, new byte[] { 1 });
s1.ChangeState(new CandidateState(s1)); // will push s1 to term 2
s2.Advance();
s1.Advance();
Assert.AreEqual(s1.ID, s2.PersistedStore.VotedFor);
Assert.AreEqual(true, s1.Clients.First().VoteGranted);
}
}