public async Task FailOverTest()
{
var dataDir = Directory.CreateDirectory(
Path.Combine(Path.GetTempPath(), "dqlite_tests_" + Guid.NewGuid()));
try
{
var builder = new DqliteConnectionStringBuilder();
builder.Nodes = new string[] { "127.0.0.1:5001", "127.0.0.1:5002", "127.0.0.1:5003" };
using (var node02 = DqliteNode.Create(2, "127.0.0.1:5002", Path.Combine(dataDir.FullName, "2")))
using (var node03 = DqliteNode.Create(3, "127.0.0.1:5003", Path.Combine(dataDir.FullName, "3")))
{
node02.Start();
node03.Start();
using (var node01 = DqliteNode.Create(1, "127.0.0.1:5001", Path.Combine(dataDir.FullName, "1")))
{
node01.Start();
using (var client = new DqliteClient(builder, true))
{
await client.ConnectAsync();
await client.AddNodeAsync(2, "127.0.0.1:5002", DqliteNodeRoles.Voter);
await client.AddNodeAsync(3, "127.0.0.1:5003", DqliteNodeRoles.Voter);
var leader = await client.GetLeaderAsync();
var nodes = await client.GetNodesAsync();
Assert.Equal(1UL, leader.Id);
Assert.Equal(3, nodes.Length);
}
}
using (var cts = new CancellationTokenSource())
using (var client = new DqliteClient(builder, true))
{
cts.CancelAfter(TimeSpan.FromMinutes(1));
await client.ConnectAsync(cts.Token);
var leader = await client.GetLeaderAsync();
var nodes = await client.GetNodesAsync();
Assert.NotEqual(1UL, leader.Id);
Assert.Equal(3, nodes.Count());
}
}
}
finally
{
dataDir.Delete(true);
}
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
Directory.CreateDirectory(options.DataDir);
this.node = DqliteNode.Create(options.Id, options.Address, options.DataDir, options.NodeOptions);
this.node.Start();
using (var client = new DqliteClient(this.options.ConnectionOptions, true))
{
await client.ConnectAsync(cancellationToken);
var nodes = await client.GetNodesAsync(cancellationToken);
var node = nodes.FirstOrDefault(x => x.Id == this.options.Id);
if (node == null)
{
await client.AddNodeAsync(this.options.Id, this.options.Address, cancellationToken);
await client.PromoteNodeAsync(this.options.Id, cancellationToken);
}
else if (node.Address != this.options.Address)
{
throw new InvalidOperationException("Node with same id already exist with different address");
}
}
await this.ExecuteAsync(x => x.StartAsync(cancellationToken));
await base.StartAsync(cancellationToken);
}
public async Task ClusterTest()
{
var dataDir = Directory.CreateDirectory(
Path.Combine(Path.GetTempPath(), "dqlite_tests_" + Guid.NewGuid()));
try
{
DqliteConnectionStringBuilder builder = new DqliteConnectionStringBuilder();
builder.Nodes = new string[] { "127.0.0.1:5001", "127.0.0.1:5002", "127.0.0.1:5003" };
using (var node01 = new NodeProcess(1, "127.0.0.1:5001", Path.Combine(dataDir.FullName, "1")))
using (var node02 = new NodeProcess(2, "127.0.0.1:5002", Path.Combine(dataDir.FullName, "2")))
using (var node03 = new NodeProcess(3, "127.0.0.1:5003", Path.Combine(dataDir.FullName, "3")))
{
Assert.False(node01.HasExited);
Assert.False(node02.HasExited);
Assert.False(node03.HasExited);
await Task.Delay(10000);
using (var client = new DqliteClient(builder))
{
await client.ConnectAsync();
await client.AddNodeAsync(2, "127.0.0.1:5002");
Assert.Equal(1ul, (await client.GetLeaderAsync()).Id);
await client.PromoteNodeAsync(2);
await client.AddNodeAsync(3, "127.0.0.1:5003");
await client.PromoteNodeAsync(3);
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
using (var client = new DqliteClient(builder))
{
await client.ConnectAsync();
var leader = await client.GetLeaderAsync();
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
using (var client = new DqliteClient(builder))
{
await client.ConnectAsync();
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
node01.Kill();
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(30 * 1000);
using (var client = new DqliteClient(builder))
{
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
}
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(30 * 1000);
using (var client = new DqliteClient(builder))
{
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
}
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(30 * 1000);
using (var newNode01 = new NodeProcess(1, "127.0.0.1:5004", Path.Combine(dataDir.FullName, "4")))
using (var client = new DqliteClient(builder))
{
Console.WriteLine("Started Removing node");
await client.RemoveNodeAsync(1);
Console.WriteLine("Finished Removing node");
await client.AddNodeAsync(1, "127.0.0.1:5004");
await client.PromoteNodeAsync(1);
var nodes = await client.GetNodesAsync();
Assert.Equal(3, nodes.Count());
}
}
}
}
finally
{
dataDir.Delete(true);
}
}