public void Non_leader_Node_removed_from_cluster_should_update_peers_list(int nodeCount)
{
var leader = CreateNetworkAndGetLeader(nodeCount);
var cmd = new DictionaryCommand.Set
{
Key = "a",
Value = 1
};
var waitForCommitsOnCluster = WaitForCommitsOnCluster(machine => machine.LastAppliedIndex == cmd.AssignedIndex);
leader.AppendCommand(cmd);
Assert.True(waitForCommitsOnCluster.Wait(3000));
var nodeToRemove = Nodes.First(x => x.State != RaftEngineState.Leader);
var nodesThatShouldRemain = Nodes.Where(n => ReferenceEquals(n, nodeToRemove) == false)
.ToList();
var waitForToplogyChangeOnCluster = WaitForToplogyChangeOnCluster(nodesThatShouldRemain);
leader.RemoveFromClusterAsync(new NodeConnectionInfo {
Name = nodeToRemove.Name
}).Wait();
Assert.True(waitForToplogyChangeOnCluster.Wait(5000));
var nodePeerLists = Nodes.Where(n => ReferenceEquals(n, nodeToRemove) == false)
.Select(n => n)
.ToList();
foreach (var node in nodePeerLists)
{
Assert.Equal(nodesThatShouldRemain.Count(), node.CurrentTopology.AllNodeNames.Count());
foreach (var n in nodesThatShouldRemain)
{
Assert.True(node.CurrentTopology.Contains(n.Name));
}
}
}
public void Network_partition_for_less_time_than_timeout_can_be_healed_without_elections(int nodeCount)
{
const int CommandCount = 5;
var commands = Builder <DictionaryCommand.Set> .CreateListOfSize(CommandCount)
.All()
.With(x => x.Completion = new TaskCompletionSource <object>())
.With(x => x.AssignedIndex = -1)
.Build()
.ToList();
var leader = CreateNetworkAndGetLeader(nodeCount, messageTimeout: 1500);
var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader);
commands.Take(CommandCount - 1).ToList().ForEach(leader.AppendCommand);
while (nonLeaderNode.CommitIndex < 2) //make sure at least one command is committed
{
Thread.Sleep(50);
}
WriteLine("<Disconnecting leader!> (" + leader.Name + ")");
DisconnectNode(leader.Name);
DictionaryCommand.Set command = commands.Last();
leader.AppendCommand(command);
var waitForCommitsOnCluster = WaitForCommitsOnCluster(machine => machine.LastAppliedIndex == command.AssignedIndex);
WriteLine("<Reconnecting leader!> (" + leader.Name + ")");
ReconnectNode(leader.Name);
Assert.Equal(RaftEngineState.Leader, leader.State);
Assert.True(waitForCommitsOnCluster.Wait(3000));
var committedCommands = nonLeaderNode.PersistentState.LogEntriesAfter(0).Select(x => nonLeaderNode.PersistentState.CommandSerializer.Deserialize(x.Data))
.OfType <DictionaryCommand.Set>()
.ToList();
for (int i = 0; i < CommandCount; i++)
{
commands[i].Value.Should().Be(committedCommands[i].Value);
commands[i].AssignedIndex.Should().Be(committedCommands[i].AssignedIndex);
}
}
public void Non_leader_Node_removed_from_cluster_should_update_peers_list(int nodeCount)
{
var leader = CreateNetworkAndGetLeader(nodeCount);
var cmd = new DictionaryCommand.Set
{
Key = "a",
Value = 1
};
var waitForCommitsOnCluster = WaitForCommitsOnCluster(machine => machine.LastAppliedIndex == cmd.AssignedIndex);
leader.AppendCommand(cmd);
Assert.True(waitForCommitsOnCluster.Wait(3000));
var nodeToRemove = Nodes.First(x => x.State != RaftEngineState.Leader);
var nodesThatShouldRemain = Nodes.Where(n => ReferenceEquals(n, nodeToRemove) == false)
.ToList();
var waitForToplogyChangeOnCluster = WaitForToplogyChangeOnCluster(nodesThatShouldRemain);
leader.RemoveFromClusterAsync(new NodeConnectionInfo { Name = nodeToRemove.Name }).Wait();
Assert.True(waitForToplogyChangeOnCluster.Wait(5000));
var nodePeerLists = Nodes.Where(n => ReferenceEquals(n, nodeToRemove) == false)
.Select(n => n)
.ToList();
foreach (var node in nodePeerLists)
{
Assert.Equal(nodesThatShouldRemain.Count(), node.CurrentTopology.AllNodeNames.Count());
foreach (var n in nodesThatShouldRemain)
{
Assert.True(node.CurrentTopology.Contains(n.Name));
}
}
}
