public static async Task Leadership(int delay)
{
void CheckLeadership(IClusterMember member1, IClusterMember member2)
=> Equal(member1.Endpoint, member2.Endpoint);
var config1 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
var config2 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
var config3 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
using (var listener1 = new LeaderChangedEvent())
using (var listener2 = new LeaderChangedEvent())
using (var listener3 = new LeaderChangedEvent())
using (var host1 = CreateHost <Startup>(3262, true, config1, listener1))
using (var host2 = CreateHost <Startup>(3263, true, config2, listener2))
using (var host3 = CreateHost <Startup>(3264, true, config3, listener3))
{
await host1.StartAsync();
await host2.StartAsync();
await Task.Delay(delay);
await host3.StartAsync();
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2, listener3 });
var leader1 = host1.Services.GetRequiredService <ICluster>().Leader;
NotNull(leader1);
var leader2 = host2.Services.GetRequiredService <ICluster>().Leader;
NotNull(leader2);
var leader3 = host3.Services.GetRequiredService <ICluster>().Leader;
NotNull(leader3);
CheckLeadership(leader1, leader2);
CheckLeadership(leader1, leader3);
listener1.Reset();
listener2.Reset();
listener3.Reset();
listener1.Leader = listener2.Leader = listener3.Leader = null;
//let's shutdown leader node
var removedNode = default(int?);
if (!leader1.IsRemote)
{
removedNode = 1;
await host1.StopAsync();
}
if (!leader2.IsRemote)
{
removedNode = 2;
await host2.StopAsync();
}
if (!leader3.IsRemote)
{
removedNode = 3;
await host3.StopAsync();
}
NotNull(removedNode);
switch (removedNode)
{
case 1:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener2, listener3 });
NotNull(listener2.Leader);
NotNull(listener3.Leader);
CheckLeadership(listener2.Leader, listener3.Leader);
break;
case 2:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener3 });
NotNull(listener1.Leader);
NotNull(listener3.Leader);
CheckLeadership(listener1.Leader, listener3.Leader);
break;
case 3:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2 });
NotNull(listener1.Leader);
NotNull(listener2.Leader);
CheckLeadership(listener1.Leader, listener2.Leader);
break;
default:
throw new Exception();
}
await host3.StopAsync();
await host2.StopAsync();
await host1.StopAsync();
}
}
public static async Task Leadership(int delay)
{
void CheckLeadership(IClusterMember member1, IClusterMember member2)
=> Equal(member1.Endpoint, member2.Endpoint);
var config1 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
var config2 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
var config3 = new Dictionary <string, string>
{
{ "partitioning", "false" },
{ "members:0", "http://localhost:3262" },
{ "members:1", "http://localhost:3263" },
{ "members:2", "http://localhost:3264" }
};
using (var listener1 = new LeaderChangedEvent())
using (var listener2 = new LeaderChangedEvent())
using (var listener3 = new LeaderChangedEvent())
using (var host1 = CreateHost <Startup>(3262, true, config1, listener1))
using (var host2 = CreateHost <Startup>(3263, true, config2, listener2))
using (var host3 = CreateHost <Startup>(3264, true, config3, listener3))
{
await host1.StartAsync();
await host2.StartAsync();
await Task.Delay(delay);
await host3.StartAsync();
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2, listener3 });
IClusterMember leader1, leader2, leader3;
//wait for stable election
for (var timer = Task.Delay(2000); ; await Task.Delay(100))
{
if (timer.IsCompleted)
{
throw new RaftProtocolException("Leader election failed");
}
leader1 = host1.Services.GetRequiredService <ICluster>().Leader;
leader2 = host2.Services.GetRequiredService <ICluster>().Leader;
leader3 = host3.Services.GetRequiredService <ICluster>().Leader;
if (leader1 is null || leader2 is null || leader3 is null)
{
continue;
}
if (leader1.Endpoint.Equals(leader2.Endpoint) && leader1.Endpoint.Equals(leader2.Endpoint))
{
break;
}
}
listener1.Reset();
listener2.Reset();
listener3.Reset();
listener1.Leader = listener2.Leader = listener3.Leader = null;
//let's shutdown leader node
var removedNode = default(int?);
if (!leader1.IsRemote)
{
removedNode = 1;
await host1.StopAsync();
}
if (!leader2.IsRemote)
{
removedNode = 2;
await host2.StopAsync();
}
if (!leader3.IsRemote)
{
removedNode = 3;
await host3.StopAsync();
}
NotNull(removedNode);
switch (removedNode)
{
case 1:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener2, listener3 });
NotNull(listener2.Leader);
NotNull(listener3.Leader);
CheckLeadership(listener2.Leader, listener3.Leader);
break;
case 2:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener3 });
NotNull(listener1.Leader);
NotNull(listener3.Leader);
CheckLeadership(listener1.Leader, listener3.Leader);
break;
case 3:
//wait for new leader
WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2 });
NotNull(listener1.Leader);
NotNull(listener2.Leader);
CheckLeadership(listener1.Leader, listener2.Leader);
break;
default:
throw new Exception();
}
//check metrics
var numberOfRequests =
(host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount +
(host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount +
(host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount;
var hasLeader = (host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator |
(host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator |
(host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator;
var heartbeats = (host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount +
(host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount +
(host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount;
True(hasLeader);
True(numberOfRequests > 0);
True(heartbeats > 0);
await host3.StopAsync();
await host2.StopAsync();
await host1.StopAsync();
}
}