public void SetUp()
{
testScheduler = new VirtualScheduler();
nodes = TestNode.CreateCluster(scheduler: testScheduler);
nodes.Start();
leader = nodes.WaitForLeader(testScheduler).Result;
}
public async Task when_a_candidate_recieves_a_vote_response_from_a_node_with_a_higher_term_number_then_that_candidate_steps_down()
{
var peerObservable = new Subject <IEnumerable <Peer> >();
var testScheduler = new VirtualScheduler();
var nodes = TestNode.CreateCluster(peerObservable: peerObservable, scheduler: testScheduler);
var leader = nodes.First();
testScheduler.AdvanceBy(TestNode.ElectionTimeout);
await leader.StartElection();
leader.StepDown(2, "none");
testScheduler.AdvanceBy(TestNode.ElectionTimeout);
nodes.LogStatus();
await leader.StartElection();
leader.State.Should().Be(State.Leader);
nodes.ForEach(n => n.Term.Should().Be(3));
var candidateThatsBehind = TestNode.CreateTestNode("candidateThatsBehind", peerObservable, testScheduler);
nodes.Add(candidateThatsBehind);
peerObservable.OnNext(nodes.Select(n => n.AsPeer()));
testScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks * 2);
await candidateThatsBehind.StartElection();
leader.State.Should().Be(State.Leader);
candidateThatsBehind.Term.Should().Be(3);
candidateThatsBehind.State.Should().Be(State.Follower, "the node should have immediatly stepped down because it attempted to start an election for term 2 but the cluster was on term 3");
}
public void SetUp()
{
testScheduler = new VirtualScheduler();
nodes = TestNode.CreateCluster(scheduler: testScheduler);
nodes.Start();
leader = nodes.WaitForLeader(testScheduler).Result;
}
public async Task when_a_candidate_recieves_a_vote_response_from_a_node_with_a_higher_term_number_then_that_candidate_steps_down()
{
var peerObservable = new Subject<IEnumerable<Peer>>();
var testScheduler = new VirtualScheduler();
var nodes = TestNode.CreateCluster(peerObservable: peerObservable, scheduler: testScheduler);
var leader = nodes.First();
testScheduler.AdvanceBy(TestNode.ElectionTimeout);
await leader.StartElection();
leader.StepDown(2, "none");
testScheduler.AdvanceBy(TestNode.ElectionTimeout);
nodes.LogStatus();
await leader.StartElection();
leader.State.Should().Be(State.Leader);
nodes.ForEach(n => n.Term.Should().Be(3));
var candidateThatsBehind = TestNode.CreateTestNode("candidateThatsBehind", peerObservable, testScheduler);
nodes.Add(candidateThatsBehind);
peerObservable.OnNext(nodes.Select(n => n.AsPeer()));
testScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks*2);
await candidateThatsBehind.StartElection();
leader.State.Should().Be(State.Leader);
candidateThatsBehind.Term.Should().Be(3);
candidateThatsBehind.State.Should().Be(State.Follower, "the node should have immediatly stepped down because it attempted to start an election for term 2 but the cluster was on term 3");
}
public async Task when_a_cluster_has_nodes_that_crash_very_often_then_logs_are_still_persisted()
{
Subject<IEnumerable<Peer>> peerObservable = new Subject<IEnumerable<Peer>>();
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 20, peerObservable: peerObservable, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
var crasher = cluster.ToArray().StartRandomlyCrashing(peerObservable, new Random(), scheduler);
var validator = Observable.Interval(TimeSpan.FromSeconds(1), scheduler)
.Subscribe(async o =>
{
Validate(cluster);
});
Enumerable.Range(1, 100).ForEach(async i =>
{
var leader = cluster.Leader();
if (leader != null)
{
await leader.AddLog(i.ToString());
expectedServerLog.Add(leader.Term, i.ToString());
}
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
await cluster.WaitForLeader(scheduler);
cluster.Leader().ServerLog.Entries.ShouldBeEquivalentTo(expectedServerLog.Entries);
}
public async Task when_a_cluster_has_nodes_that_crash_very_often_then_logs_are_still_persisted()
{
Subject <IEnumerable <Peer> > peerObservable = new Subject <IEnumerable <Peer> >();
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 20, peerObservable: peerObservable, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
var crasher = cluster.ToArray().StartRandomlyCrashing(peerObservable, new Random(), scheduler);
var validator = Observable.Interval(TimeSpan.FromSeconds(1), scheduler)
.Subscribe(async o =>
{
Validate(cluster);
});
Enumerable.Range(1, 100).ForEach(async i =>
{
var leader = cluster.Leader();
if (leader != null)
{
await leader.AddLog(i.ToString());
expectedServerLog.Add(leader.Term, i.ToString());
}
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
await cluster.WaitForLeader(scheduler);
cluster.Leader().ServerLog.Entries.ShouldBeEquivalentTo(expectedServerLog.Entries);
}
public async Task clusters_remain_in_a_valid_state_throughout_logging()
{
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 2, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
var validationCounter = 0;
Observable.Interval(TimeSpan.FromSeconds(1), scheduler)
.Subscribe(async o =>
{
Validate(cluster);
validationCounter++;
});
Enumerable.Range(1, 100).ForEach(async i =>
{
await cluster.Leader().AddLog(i.ToString());
expectedServerLog.Add(cluster.Leader().Term, i.ToString());
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
Console.WriteLine("Validation ran {0} times on cluster", validationCounter);
}
public async Task logs_can_be_quickly_written_in_parallel_to_a_leader_with_test_scheduler()
{
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 4, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
Enumerable.Range(1, 100).ForEach(i =>
{
cluster.Leader().AddLog(i.ToString()).Wait();
expectedServerLog.Add(cluster.Leader().Term, i.ToString());
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
cluster.Leader().LocalLog.Entries.Count.Should().Be(101);
cluster.Leader().ServerLog.Entries.Count.Should().Be(101);
cluster.Followers().ForEach(f =>
{
Enumerable.Range(1, 100).ForEach(i =>
{
var s = i.ToString();
f.LocalLog.Entries.Select(e => e.Log).Should().Contain(s);
});
});
}
public void elections_are_ran_shortly_after_node_startup()
{
var virtualScheduler = new VirtualScheduler();
var nodes = TestNode.CreateCluster(scheduler: virtualScheduler);
nodes.Start();
virtualScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks);
nodes.Should().Contain(n => n.State == State.Leader || n.State == State.Candidate);
}
public void SetUp()
{
testScheduler = new VirtualScheduler();
nodes = TestNode.CreateCluster(scheduler: testScheduler);
nodes.Start();
leader = nodes.WaitForLeader(testScheduler).Result;
darkNode = nodes.First(n => n != leader) as TestNode;
darkNode.SleepOnAppendEntries = true;
}
public void elections_are_ran_shortly_after_node_startup()
{
var virtualScheduler = new VirtualScheduler();
var nodes = TestNode.CreateCluster(scheduler: virtualScheduler);
nodes.Start();
virtualScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks);
nodes.Should().Contain(n => n.State == State.Leader || n.State == State.Candidate);
}
public void SetUp()
{
testScheduler = new VirtualScheduler();
nodes = TestNode.CreateCluster(scheduler: testScheduler);
nodes.Start();
leader = nodes.WaitForLeader(testScheduler).Result;
darkNode = nodes.First(n => n != leader) as TestNode;
darkNode.SleepOnAppendEntries = true;
}
public void SetUp()
{
scheduler = new VirtualScheduler();
leftBrainPeerObservable = new Subject <IEnumerable <Peer> >();
rightBrainPeerObservable = new Subject <IEnumerable <Peer> >();
leftBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: leftBrainPeerObservable, scheduler: scheduler);
rightBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: rightBrainPeerObservable, scheduler: scheduler);
leftBrainLeader = leftBrain.First();
leftBrainLeader.StartElection().Wait();
rightBrain.First().StartElection().Wait();
rightBrain.First().StepDown(2, "none");
scheduler.AdvanceBy(TestNode.ElectionTimeout);
rightBrain.First().StartElection().Wait();
rightBrain.ForEach(n => n.Term.Should().Be(3));
}
public void SetUp()
{
scheduler = new VirtualScheduler();
leftBrainPeerObservable = new Subject<IEnumerable<Peer>>();
rightBrainPeerObservable = new Subject<IEnumerable<Peer>>();
leftBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: leftBrainPeerObservable, scheduler: scheduler);
rightBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: rightBrainPeerObservable, scheduler: scheduler);
leftBrainLeader = leftBrain.First();
leftBrainLeader.StartElection().Wait();
rightBrain.First().StartElection().Wait();
rightBrain.First().StepDown(2, "none");
scheduler.AdvanceBy(TestNode.ElectionTimeout);
rightBrain.First().StartElection().Wait();
rightBrain.ForEach(n => n.Term.Should().Be(3));
}
public async Task clusters_remain_in_a_valid_state_throughout_logging()
{
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 2, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
var validationCounter = 0;
Observable.Interval(TimeSpan.FromSeconds(1), scheduler)
.Subscribe(async o =>
{
Validate(cluster);
validationCounter ++;
});
Enumerable.Range(1, 100).ForEach(async i =>
{
await cluster.Leader().AddLog(i.ToString());
expectedServerLog.Add(cluster.Leader().Term, i.ToString());
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
Console.WriteLine("Validation ran {0} times on cluster", validationCounter);
}
public async Task logs_can_be_quickly_written_in_parallel_to_a_leader_with_test_scheduler()
{
var scheduler = new VirtualScheduler();
var cluster = TestNode.CreateCluster(clusterSize: 4, scheduler: scheduler);
cluster.Start();
await cluster.WaitForLeader(scheduler);
var expectedServerLog = new Log();
Enumerable.Range(1, 100).ForEach( i =>
{
cluster.Leader().AddLog(i.ToString()).Wait();
expectedServerLog.Add(cluster.Leader().Term, i.ToString());
scheduler.AdvanceBy(TimeSpan.FromSeconds(1));
});
cluster.Leader().LocalLog.Entries.Count.Should().Be(101);
cluster.Leader().ServerLog.Entries.Count.Should().Be(101);
cluster.Followers().ForEach(f =>
{
Enumerable.Range(1, 100).ForEach(i =>
{
var s = i.ToString();
f.LocalLog.Entries.Select(e => e.Log).Should().Contain(s);
});
});
}