public static async Task Leadership(int delay)
        {
            void CheckLeadership(IClusterMember member1, IClusterMember member2)
            => Equal(member1.Endpoint, member2.Endpoint);

            var config1 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };
            var config2 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };
            var config3 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };

            using (var listener1 = new LeaderChangedEvent())
                using (var listener2 = new LeaderChangedEvent())
                    using (var listener3 = new LeaderChangedEvent())
                        using (var host1 = CreateHost <Startup>(3262, true, config1, listener1))
                            using (var host2 = CreateHost <Startup>(3263, true, config2, listener2))
                                using (var host3 = CreateHost <Startup>(3264, true, config3, listener3))
                                {
                                    await host1.StartAsync();

                                    await host2.StartAsync();

                                    await Task.Delay(delay);

                                    await host3.StartAsync();

                                    WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2, listener3 });

                                    var leader1 = host1.Services.GetRequiredService <ICluster>().Leader;
                                    NotNull(leader1);
                                    var leader2 = host2.Services.GetRequiredService <ICluster>().Leader;
                                    NotNull(leader2);
                                    var leader3 = host3.Services.GetRequiredService <ICluster>().Leader;
                                    NotNull(leader3);

                                    CheckLeadership(leader1, leader2);
                                    CheckLeadership(leader1, leader3);

                                    listener1.Reset();
                                    listener2.Reset();
                                    listener3.Reset();
                                    listener1.Leader = listener2.Leader = listener3.Leader = null;

                                    //let's shutdown leader node

                                    var removedNode = default(int?);

                                    if (!leader1.IsRemote)
                                    {
                                        removedNode = 1;
                                        await host1.StopAsync();
                                    }

                                    if (!leader2.IsRemote)
                                    {
                                        removedNode = 2;
                                        await host2.StopAsync();
                                    }

                                    if (!leader3.IsRemote)
                                    {
                                        removedNode = 3;
                                        await host3.StopAsync();
                                    }

                                    NotNull(removedNode);

                                    switch (removedNode)
                                    {
                                    case 1:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener2, listener3 });
                                        NotNull(listener2.Leader);
                                        NotNull(listener3.Leader);
                                        CheckLeadership(listener2.Leader, listener3.Leader);
                                        break;

                                    case 2:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener1, listener3 });
                                        NotNull(listener1.Leader);
                                        NotNull(listener3.Leader);
                                        CheckLeadership(listener1.Leader, listener3.Leader);
                                        break;

                                    case 3:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2 });
                                        NotNull(listener1.Leader);
                                        NotNull(listener2.Leader);
                                        CheckLeadership(listener1.Leader, listener2.Leader);
                                        break;

                                    default:
                                        throw new Exception();
                                    }

                                    await host3.StopAsync();

                                    await host2.StopAsync();

                                    await host1.StopAsync();
                                }
        }
        public static async Task Leadership(int delay)
        {
            void CheckLeadership(IClusterMember member1, IClusterMember member2)
            => Equal(member1.Endpoint, member2.Endpoint);

            var config1 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };
            var config2 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };
            var config3 = new Dictionary <string, string>
            {
                { "partitioning", "false" },
                { "members:0", "http://localhost:3262" },
                { "members:1", "http://localhost:3263" },
                { "members:2", "http://localhost:3264" }
            };

            using (var listener1 = new LeaderChangedEvent())
                using (var listener2 = new LeaderChangedEvent())
                    using (var listener3 = new LeaderChangedEvent())
                        using (var host1 = CreateHost <Startup>(3262, true, config1, listener1))
                            using (var host2 = CreateHost <Startup>(3263, true, config2, listener2))
                                using (var host3 = CreateHost <Startup>(3264, true, config3, listener3))
                                {
                                    await host1.StartAsync();

                                    await host2.StartAsync();

                                    await Task.Delay(delay);

                                    await host3.StartAsync();

                                    WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2, listener3 });

                                    IClusterMember leader1, leader2, leader3;

                                    //wait for stable election
                                    for (var timer = Task.Delay(2000); ; await Task.Delay(100))
                                    {
                                        if (timer.IsCompleted)
                                        {
                                            throw new RaftProtocolException("Leader election failed");
                                        }
                                        leader1 = host1.Services.GetRequiredService <ICluster>().Leader;
                                        leader2 = host2.Services.GetRequiredService <ICluster>().Leader;
                                        leader3 = host3.Services.GetRequiredService <ICluster>().Leader;
                                        if (leader1 is null || leader2 is null || leader3 is null)
                                        {
                                            continue;
                                        }
                                        if (leader1.Endpoint.Equals(leader2.Endpoint) && leader1.Endpoint.Equals(leader2.Endpoint))
                                        {
                                            break;
                                        }
                                    }

                                    listener1.Reset();
                                    listener2.Reset();
                                    listener3.Reset();
                                    listener1.Leader = listener2.Leader = listener3.Leader = null;

                                    //let's shutdown leader node

                                    var removedNode = default(int?);

                                    if (!leader1.IsRemote)
                                    {
                                        removedNode = 1;
                                        await host1.StopAsync();
                                    }

                                    if (!leader2.IsRemote)
                                    {
                                        removedNode = 2;
                                        await host2.StopAsync();
                                    }

                                    if (!leader3.IsRemote)
                                    {
                                        removedNode = 3;
                                        await host3.StopAsync();
                                    }

                                    NotNull(removedNode);

                                    switch (removedNode)
                                    {
                                    case 1:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener2, listener3 });
                                        NotNull(listener2.Leader);
                                        NotNull(listener3.Leader);
                                        CheckLeadership(listener2.Leader, listener3.Leader);
                                        break;

                                    case 2:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener1, listener3 });
                                        NotNull(listener1.Leader);
                                        NotNull(listener3.Leader);
                                        CheckLeadership(listener1.Leader, listener3.Leader);
                                        break;

                                    case 3:
                                        //wait for new leader
                                        WaitHandle.WaitAll(new WaitHandle[] { listener1, listener2 });
                                        NotNull(listener1.Leader);
                                        NotNull(listener2.Leader);
                                        CheckLeadership(listener1.Leader, listener2.Leader);
                                        break;

                                    default:
                                        throw new Exception();
                                    }
                                    //check metrics
                                    var numberOfRequests =
                                        (host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount +
                                        (host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount +
                                        (host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).RequestCount;

                                    var hasLeader = (host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator |
                                                    (host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator |
                                                    (host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).LeaderStateIndicator;

                                    var heartbeats = (host1.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount +
                                                     (host2.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount +
                                                     (host3.Services.GetService <MetricsCollector>() as TestMetricsCollector).HeartbeatCount;

                                    True(hasLeader);
                                    True(numberOfRequests > 0);
                                    True(heartbeats > 0);

                                    await host3.StopAsync();

                                    await host2.StopAsync();

                                    await host1.StopAsync();
                                }
        }