public void SetUp() { testScheduler = new VirtualScheduler(); nodes = TestNode.CreateCluster(scheduler: testScheduler); nodes.Start(); leader = nodes.WaitForLeader(testScheduler).Result; }
public async Task when_a_candidate_recieves_a_vote_response_from_a_node_with_a_higher_term_number_then_that_candidate_steps_down() { var peerObservable = new Subject <IEnumerable <Peer> >(); var testScheduler = new VirtualScheduler(); var nodes = TestNode.CreateCluster(peerObservable: peerObservable, scheduler: testScheduler); var leader = nodes.First(); testScheduler.AdvanceBy(TestNode.ElectionTimeout); await leader.StartElection(); leader.StepDown(2, "none"); testScheduler.AdvanceBy(TestNode.ElectionTimeout); nodes.LogStatus(); await leader.StartElection(); leader.State.Should().Be(State.Leader); nodes.ForEach(n => n.Term.Should().Be(3)); var candidateThatsBehind = TestNode.CreateTestNode("candidateThatsBehind", peerObservable, testScheduler); nodes.Add(candidateThatsBehind); peerObservable.OnNext(nodes.Select(n => n.AsPeer())); testScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks * 2); await candidateThatsBehind.StartElection(); leader.State.Should().Be(State.Leader); candidateThatsBehind.Term.Should().Be(3); candidateThatsBehind.State.Should().Be(State.Follower, "the node should have immediatly stepped down because it attempted to start an election for term 2 but the cluster was on term 3"); }
public void SetUp() { testScheduler = new VirtualScheduler(); nodes = TestNode.CreateCluster(scheduler: testScheduler); nodes.Start(); leader = nodes.WaitForLeader(testScheduler).Result; }
public async Task when_a_candidate_recieves_a_vote_response_from_a_node_with_a_higher_term_number_then_that_candidate_steps_down() { var peerObservable = new Subject<IEnumerable<Peer>>(); var testScheduler = new VirtualScheduler(); var nodes = TestNode.CreateCluster(peerObservable: peerObservable, scheduler: testScheduler); var leader = nodes.First(); testScheduler.AdvanceBy(TestNode.ElectionTimeout); await leader.StartElection(); leader.StepDown(2, "none"); testScheduler.AdvanceBy(TestNode.ElectionTimeout); nodes.LogStatus(); await leader.StartElection(); leader.State.Should().Be(State.Leader); nodes.ForEach(n => n.Term.Should().Be(3)); var candidateThatsBehind = TestNode.CreateTestNode("candidateThatsBehind", peerObservable, testScheduler); nodes.Add(candidateThatsBehind); peerObservable.OnNext(nodes.Select(n => n.AsPeer())); testScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks*2); await candidateThatsBehind.StartElection(); leader.State.Should().Be(State.Leader); candidateThatsBehind.Term.Should().Be(3); candidateThatsBehind.State.Should().Be(State.Follower, "the node should have immediatly stepped down because it attempted to start an election for term 2 but the cluster was on term 3"); }
public async Task when_a_cluster_has_nodes_that_crash_very_often_then_logs_are_still_persisted() { Subject<IEnumerable<Peer>> peerObservable = new Subject<IEnumerable<Peer>>(); var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 20, peerObservable: peerObservable, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); var crasher = cluster.ToArray().StartRandomlyCrashing(peerObservable, new Random(), scheduler); var validator = Observable.Interval(TimeSpan.FromSeconds(1), scheduler) .Subscribe(async o => { Validate(cluster); }); Enumerable.Range(1, 100).ForEach(async i => { var leader = cluster.Leader(); if (leader != null) { await leader.AddLog(i.ToString()); expectedServerLog.Add(leader.Term, i.ToString()); } scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); await cluster.WaitForLeader(scheduler); cluster.Leader().ServerLog.Entries.ShouldBeEquivalentTo(expectedServerLog.Entries); }
public async Task when_a_cluster_has_nodes_that_crash_very_often_then_logs_are_still_persisted() { Subject <IEnumerable <Peer> > peerObservable = new Subject <IEnumerable <Peer> >(); var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 20, peerObservable: peerObservable, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); var crasher = cluster.ToArray().StartRandomlyCrashing(peerObservable, new Random(), scheduler); var validator = Observable.Interval(TimeSpan.FromSeconds(1), scheduler) .Subscribe(async o => { Validate(cluster); }); Enumerable.Range(1, 100).ForEach(async i => { var leader = cluster.Leader(); if (leader != null) { await leader.AddLog(i.ToString()); expectedServerLog.Add(leader.Term, i.ToString()); } scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); await cluster.WaitForLeader(scheduler); cluster.Leader().ServerLog.Entries.ShouldBeEquivalentTo(expectedServerLog.Entries); }
public async Task clusters_remain_in_a_valid_state_throughout_logging() { var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 2, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); var validationCounter = 0; Observable.Interval(TimeSpan.FromSeconds(1), scheduler) .Subscribe(async o => { Validate(cluster); validationCounter++; }); Enumerable.Range(1, 100).ForEach(async i => { await cluster.Leader().AddLog(i.ToString()); expectedServerLog.Add(cluster.Leader().Term, i.ToString()); scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); Console.WriteLine("Validation ran {0} times on cluster", validationCounter); }
public async Task logs_can_be_quickly_written_in_parallel_to_a_leader_with_test_scheduler() { var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 4, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); Enumerable.Range(1, 100).ForEach(i => { cluster.Leader().AddLog(i.ToString()).Wait(); expectedServerLog.Add(cluster.Leader().Term, i.ToString()); scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); cluster.Leader().LocalLog.Entries.Count.Should().Be(101); cluster.Leader().ServerLog.Entries.Count.Should().Be(101); cluster.Followers().ForEach(f => { Enumerable.Range(1, 100).ForEach(i => { var s = i.ToString(); f.LocalLog.Entries.Select(e => e.Log).Should().Contain(s); }); }); }
public void elections_are_ran_shortly_after_node_startup() { var virtualScheduler = new VirtualScheduler(); var nodes = TestNode.CreateCluster(scheduler: virtualScheduler); nodes.Start(); virtualScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks); nodes.Should().Contain(n => n.State == State.Leader || n.State == State.Candidate); }
public void SetUp() { testScheduler = new VirtualScheduler(); nodes = TestNode.CreateCluster(scheduler: testScheduler); nodes.Start(); leader = nodes.WaitForLeader(testScheduler).Result; darkNode = nodes.First(n => n != leader) as TestNode; darkNode.SleepOnAppendEntries = true; }
public void elections_are_ran_shortly_after_node_startup() { var virtualScheduler = new VirtualScheduler(); var nodes = TestNode.CreateCluster(scheduler: virtualScheduler); nodes.Start(); virtualScheduler.AdvanceBy(TestNode.ElectionTimeout.Ticks); nodes.Should().Contain(n => n.State == State.Leader || n.State == State.Candidate); }
public void SetUp() { testScheduler = new VirtualScheduler(); nodes = TestNode.CreateCluster(scheduler: testScheduler); nodes.Start(); leader = nodes.WaitForLeader(testScheduler).Result; darkNode = nodes.First(n => n != leader) as TestNode; darkNode.SleepOnAppendEntries = true; }
public void SetUp() { scheduler = new VirtualScheduler(); leftBrainPeerObservable = new Subject <IEnumerable <Peer> >(); rightBrainPeerObservable = new Subject <IEnumerable <Peer> >(); leftBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: leftBrainPeerObservable, scheduler: scheduler); rightBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: rightBrainPeerObservable, scheduler: scheduler); leftBrainLeader = leftBrain.First(); leftBrainLeader.StartElection().Wait(); rightBrain.First().StartElection().Wait(); rightBrain.First().StepDown(2, "none"); scheduler.AdvanceBy(TestNode.ElectionTimeout); rightBrain.First().StartElection().Wait(); rightBrain.ForEach(n => n.Term.Should().Be(3)); }
public void SetUp() { scheduler = new VirtualScheduler(); leftBrainPeerObservable = new Subject<IEnumerable<Peer>>(); rightBrainPeerObservable = new Subject<IEnumerable<Peer>>(); leftBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: leftBrainPeerObservable, scheduler: scheduler); rightBrain = TestNode.CreateCluster(clusterSize: 5, peerObservable: rightBrainPeerObservable, scheduler: scheduler); leftBrainLeader = leftBrain.First(); leftBrainLeader.StartElection().Wait(); rightBrain.First().StartElection().Wait(); rightBrain.First().StepDown(2, "none"); scheduler.AdvanceBy(TestNode.ElectionTimeout); rightBrain.First().StartElection().Wait(); rightBrain.ForEach(n => n.Term.Should().Be(3)); }
public async Task clusters_remain_in_a_valid_state_throughout_logging() { var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 2, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); var validationCounter = 0; Observable.Interval(TimeSpan.FromSeconds(1), scheduler) .Subscribe(async o => { Validate(cluster); validationCounter ++; }); Enumerable.Range(1, 100).ForEach(async i => { await cluster.Leader().AddLog(i.ToString()); expectedServerLog.Add(cluster.Leader().Term, i.ToString()); scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); Console.WriteLine("Validation ran {0} times on cluster", validationCounter); }
public async Task logs_can_be_quickly_written_in_parallel_to_a_leader_with_test_scheduler() { var scheduler = new VirtualScheduler(); var cluster = TestNode.CreateCluster(clusterSize: 4, scheduler: scheduler); cluster.Start(); await cluster.WaitForLeader(scheduler); var expectedServerLog = new Log(); Enumerable.Range(1, 100).ForEach( i => { cluster.Leader().AddLog(i.ToString()).Wait(); expectedServerLog.Add(cluster.Leader().Term, i.ToString()); scheduler.AdvanceBy(TimeSpan.FromSeconds(1)); }); cluster.Leader().LocalLog.Entries.Count.Should().Be(101); cluster.Leader().ServerLog.Entries.Count.Should().Be(101); cluster.Followers().ForEach(f => { Enumerable.Range(1, 100).ForEach(i => { var s = i.ToString(); f.LocalLog.Entries.Select(e => e.Log).Should().Contain(s); }); }); }