public async Task RemoveEntryFromRaftLogEP() { var leader = await CreateRaftClusterAndGetLeader(3); var database = GetDatabaseName(); await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null); using (var store = new DocumentStore { Database = database, Urls = new[] { leader.WebUrl } }.Initialize()) { leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator(); var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId()); _ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000)); var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1); _ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString()) { Record = new DatabaseRecord("Toli") { Topology = new DatabaseTopology { Members = new List <string> { "A", "B", "C" }, Rehabs = new List <string> { }, ReplicationFactor = 3 } }, Name = "Toli" }); foreach (var server in Servers) { Assert.False(leader.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli")); } long index; using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { leader.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term); } var operation = await store.Maintenance.SendAsync(new RemoveEntryFromRaftLogOperation(index + 1)); foreach (var server in Servers) { Assert.Contains(server.ServerStore.NodeTag, operation); var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true); Assert.True(val.Result); } } }
public async Task RemoveEntryFromRaftLogEP() { var(_, leader) = await CreateRaftCluster(3, watcherCluster : true); var database = GetDatabaseName(); await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null); using (var store = new DocumentStore { Database = database, Urls = new[] { leader.WebUrl } }.Initialize()) { leader.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator(); var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId()); await Assert.ThrowsAsync <UnknownClusterCommand>(() => leader.ServerStore.SendToLeaderAsync(cmd)); var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1); _ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString()) { Record = new DatabaseRecord("Toli") { Topology = new DatabaseTopology { Members = new List <string> { "A", "B", "C" }, Rehabs = new List <string> { }, ReplicationFactor = 3 } }, Name = "Toli" }); foreach (var server in Servers) { Assert.False(server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli")); } var index = Cluster.LastRaftIndexForCommand(leader, nameof(TestCommandWithRaftId)); List <string> nodelist = new List <string>(); var res = await WaitForValueAsync(async() => { nodelist = await store.Maintenance.SendAsync(new RemoveEntryFromRaftLogOperation(index)); return(nodelist.Count); }, 3); Assert.Equal(3, res); long index2 = 0; foreach (var server in Servers) { await WaitForValueAsync(async() => { var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { server.ServerStore.Engine.GetLastCommitIndex(context, out index2, out long term); } return(index2 > index); }, true); } Assert.True(index2 > index, $"State machine is stuck. raft index was {index}, after remove raft entry index is {index2} "); foreach (var server in Servers) { var val = WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true); Assert.True(val.Result); Assert.Contains(server.ServerStore.NodeTag, nodelist); } } }
public async Task RemoveEntryFromRaftLogTest() { var(_, leader) = await CreateRaftCluster(3, watcherCluster : true); var database = GetDatabaseName(); await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null); using (var store = new DocumentStore { Database = database, Urls = new[] { leader.WebUrl } }.Initialize()) { await ActionWithLeader(l => l.ServerStore.Engine.StateMachine.Validator = new TestCommandValidator()); var documentDatabase = await leader.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); var cmd = new RachisConsensusTestBase.TestCommandWithRaftId("test", RaftIdGenerator.NewId()); _ = leader.ServerStore.Engine.CurrentLeader.PutAsync(cmd, new TimeSpan(2000)); var cmd2 = new CreateDatabaseOperation.CreateDatabaseCommand(new DatabaseRecord("Toli"), 1); _ = leader.ServerStore.SendToLeaderAsync(new AddDatabaseCommand(Guid.NewGuid().ToString()) { Record = new DatabaseRecord("Toli") { Topology = new DatabaseTopology { Members = new List <string> { "A", "B", "C" }, Rehabs = new List <string> { }, ReplicationFactor = 3 } }, Name = "Toli" }); foreach (var server in Servers) { Assert.False(server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli")); } long index = 0; foreach (var server in Servers) { index = 0; documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { server.ServerStore.Engine.GetLastCommitIndex(context, out index, out long term); } var res = await WaitForValueAsync(() => server.ServerStore.Engine.RemoveEntryFromRaftLog(index + 1), true); Assert.True(res); } long index2 = 0; foreach (var server in Servers) { await WaitForValueAsync(async() => { index2 = 0; documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (documentDatabase.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context)) using (var tx = context.OpenReadTransaction()) { server.ServerStore.Engine.GetLastCommitIndex(context, out index2, out long term); } return(index2 > index); }, true); } Assert.True(index2 > index, $"State machine is stuck. raft index was {index}, after remove raft entry index is {index2} "); foreach (var server in Servers) { var val = await WaitForValueAsync(() => server.ServerStore.DatabasesLandlord.IsDatabaseLoaded("Toli"), true); Assert.True(val); } } }