public async Task SubscriptionsShouldFailoverCorrectrlyAndAllowThemselvesToBeTerminated(int subscriptionsChainSize, int clusterSize, int dBGroupSize, bool shouldTrapRevivedNodesIntoCandidate)
{
const int SubscriptionsCount = 10;
const int DocsBatchSize = 10;
var cluster = await CreateRaftCluster(clusterSize, shouldRunInMemory : false);
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
using (var cdeArray = new CountdownsArray(subscriptionsChainSize, SubscriptionsCount))
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = dBGroupSize,
ModifyDocumentStore = s => s.Conventions.ReadBalanceBehavior = Raven.Client.Http.ReadBalanceBehavior.RoundRobin,
RunInMemory = false
}))
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User(), cts.Token);
await session.SaveChangesAsync(cts.Token);
}
var databaseName = store.Database;
var workerTasks = new List <Task>();
for (var i = 0; i < SubscriptionsCount; i++)
{
await GenerateWaitingSubscriptions(cdeArray.GetArray(), store, i, workerTasks, cluster.Nodes, cts.Token);
}
_ = Task.Run(async() => await ContinuouslyGenerateDocs(DocsBatchSize, store, cts.Token), cts.Token);
var dbNodesCountToToggle = Math.Max(Math.Min(dBGroupSize - 1, dBGroupSize / 2 + 1), 1);
var nodesToToggle = store.GetRequestExecutor().TopologyNodes.Select(x => x.ClusterTag).Take(dbNodesCountToToggle).ToList();
_toggled = true;
foreach (var node in nodesToToggle)
{
cts.Token.ThrowIfCancellationRequested();
var nodeIndex = cluster.Nodes.FindIndex(x => x.ServerStore.NodeTag == node);
await ToggleClusterNodeOnAndOffAndWaitForRehab(databaseName, cluster, nodeIndex, shouldTrapRevivedNodesIntoCandidate, cts.Token);
}
_toggled = false;
Assert.All(cdeArray.GetArray(), cde => Assert.True(SubscriptionsCount == cde.CurrentCount, PrintTestInfo(nodesToToggle, cluster)));
foreach (var cde in cdeArray.GetArray())
{
await KeepDroppingSubscriptionsAndWaitingForCDE(databaseName, SubscriptionsCount, cluster, cde, cts.Token);
}
foreach (var curNode in cluster.Nodes)
{
await AssertNoSubscriptionLeftAlive(databaseName, SubscriptionsCount, curNode, cts.Token);
}
await Task.WhenAll(workerTasks);
}
}
public async Task SubscriptionsShouldFailoverCorrectrlyAndAllowThemselvesToBeTerminated(int subscriptionsChainSize, int clusterSize, int dBGroupSize, bool shouldTrapRevivedNodesIntoCandidate)
{
const int SubscriptionsCount = 10;
const int DocsBatchSize = 10;
var cluster = await CreateRaftCluster(clusterSize, shouldRunInMemory : false);
using (var cdeArray = new CountdownsArray(subscriptionsChainSize, SubscriptionsCount))
using (var store = GetDocumentStore(new Options
{
Server = cluster.Leader,
ReplicationFactor = dBGroupSize,
ModifyDocumentStore = s => s.Conventions.ReadBalanceBehavior = Raven.Client.Http.ReadBalanceBehavior.RoundRobin
}))
{
using (var session = store.OpenSession())
{
session.Store(new User());
session.SaveChanges();
}
var databaseName = store.Database;
var workerTasks = new List <Task>();
for (var i = 0; i < SubscriptionsCount; i++)
{
await GenerateWaitingSubscriptions(cdeArray.GetArray(), store, i, workerTasks);
}
var task = Task.Run(async() =>
{
await ContinuouslyGenerateDocs(DocsBatchSize, store);
});
var dbNodesCountToToggle = Math.Max(Math.Min(dBGroupSize - 1, dBGroupSize / 2 + 1), 1);
foreach (var node in store.GetRequestExecutor().TopologyNodes.Take(dbNodesCountToToggle))
{
var i = 0;
for (; i < cluster.Nodes.Count; i++)
{
if (cluster.Nodes[i].ServerStore.NodeTag == node.ClusterTag)
{
break;
}
}
await ToggleClusterNodeOnAndOffAndWaitForRehab(databaseName, cluster, i, shouldTrapRevivedNodesIntoCandidate);
}
Assert.All(cdeArray.GetArray(), cde => Assert.Equal(cde.CurrentCount, SubscriptionsCount));
foreach (var cde in cdeArray.GetArray())
{
await KeepDroppingSubscriptionsAndWaitingForCDE(databaseName, SubscriptionsCount, cluster, cde);
}
foreach (var curNode in cluster.Nodes)
{
await AssertNoSubscriptionLeftAlive(databaseName, SubscriptionsCount, curNode);
}
await Task.WhenAll(workerTasks);
}
}
