public async Task SubscriptionsShouldFailoverCorrectrlyAndAllowThemselvesToBeTerminated(int subscriptionsChainSize, int clusterSize, int dBGroupSize, bool shouldTrapRevivedNodesIntoCandidate) { const int SubscriptionsCount = 10; const int DocsBatchSize = 10; var cluster = await CreateRaftCluster(clusterSize, shouldRunInMemory : false); using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5))) using (var cdeArray = new CountdownsArray(subscriptionsChainSize, SubscriptionsCount)) using (var store = GetDocumentStore(new Options { Server = cluster.Leader, ReplicationFactor = dBGroupSize, ModifyDocumentStore = s => s.Conventions.ReadBalanceBehavior = Raven.Client.Http.ReadBalanceBehavior.RoundRobin, RunInMemory = false })) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User(), cts.Token); await session.SaveChangesAsync(cts.Token); } var databaseName = store.Database; var workerTasks = new List <Task>(); for (var i = 0; i < SubscriptionsCount; i++) { await GenerateWaitingSubscriptions(cdeArray.GetArray(), store, i, workerTasks, cluster.Nodes, cts.Token); } _ = Task.Run(async() => await ContinuouslyGenerateDocs(DocsBatchSize, store, cts.Token), cts.Token); var dbNodesCountToToggle = Math.Max(Math.Min(dBGroupSize - 1, dBGroupSize / 2 + 1), 1); var nodesToToggle = store.GetRequestExecutor().TopologyNodes.Select(x => x.ClusterTag).Take(dbNodesCountToToggle).ToList(); _toggled = true; foreach (var node in nodesToToggle) { cts.Token.ThrowIfCancellationRequested(); var nodeIndex = cluster.Nodes.FindIndex(x => x.ServerStore.NodeTag == node); await ToggleClusterNodeOnAndOffAndWaitForRehab(databaseName, cluster, nodeIndex, shouldTrapRevivedNodesIntoCandidate, cts.Token); } _toggled = false; Assert.All(cdeArray.GetArray(), cde => Assert.True(SubscriptionsCount == cde.CurrentCount, PrintTestInfo(nodesToToggle, cluster))); foreach (var cde in cdeArray.GetArray()) { await KeepDroppingSubscriptionsAndWaitingForCDE(databaseName, SubscriptionsCount, cluster, cde, cts.Token); } foreach (var curNode in cluster.Nodes) { await AssertNoSubscriptionLeftAlive(databaseName, SubscriptionsCount, curNode, cts.Token); } await Task.WhenAll(workerTasks); } }
public async Task SubscriptionsShouldFailoverCorrectrlyAndAllowThemselvesToBeTerminated(int subscriptionsChainSize, int clusterSize, int dBGroupSize, bool shouldTrapRevivedNodesIntoCandidate) { const int SubscriptionsCount = 10; const int DocsBatchSize = 10; var cluster = await CreateRaftCluster(clusterSize, shouldRunInMemory : false); using (var cdeArray = new CountdownsArray(subscriptionsChainSize, SubscriptionsCount)) using (var store = GetDocumentStore(new Options { Server = cluster.Leader, ReplicationFactor = dBGroupSize, ModifyDocumentStore = s => s.Conventions.ReadBalanceBehavior = Raven.Client.Http.ReadBalanceBehavior.RoundRobin })) { using (var session = store.OpenSession()) { session.Store(new User()); session.SaveChanges(); } var databaseName = store.Database; var workerTasks = new List <Task>(); for (var i = 0; i < SubscriptionsCount; i++) { await GenerateWaitingSubscriptions(cdeArray.GetArray(), store, i, workerTasks); } var task = Task.Run(async() => { await ContinuouslyGenerateDocs(DocsBatchSize, store); }); var dbNodesCountToToggle = Math.Max(Math.Min(dBGroupSize - 1, dBGroupSize / 2 + 1), 1); foreach (var node in store.GetRequestExecutor().TopologyNodes.Take(dbNodesCountToToggle)) { var i = 0; for (; i < cluster.Nodes.Count; i++) { if (cluster.Nodes[i].ServerStore.NodeTag == node.ClusterTag) { break; } } await ToggleClusterNodeOnAndOffAndWaitForRehab(databaseName, cluster, i, shouldTrapRevivedNodesIntoCandidate); } Assert.All(cdeArray.GetArray(), cde => Assert.Equal(cde.CurrentCount, SubscriptionsCount)); foreach (var cde in cdeArray.GetArray()) { await KeepDroppingSubscriptionsAndWaitingForCDE(databaseName, SubscriptionsCount, cluster, cde); } foreach (var curNode in cluster.Nodes) { await AssertNoSubscriptionLeftAlive(databaseName, SubscriptionsCount, curNode); } await Task.WhenAll(workerTasks); } }