//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void lastTxCommitTimestampShouldGetInitializedOnSlaveIfNotPresent() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void LastTxCommitTimestampShouldGetInitializedOnSlaveIfNotPresent()
{
ClusterManager clusterManager = (new ClusterManager.Builder(TestDirectory.directory(TestName.MethodName))).withCluster(ClusterManager.clusterOfSize(3)).build();
try
{
clusterManager.Start();
ClusterManager.ManagedCluster cluster = clusterManager.Cluster;
cluster.Await(allSeesAllAsAvailable());
RunSomeTransactions(cluster.Master);
cluster.Sync();
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
DatabaseLayout databaseLayout = slave.DatabaseLayout();
ClusterManager.RepairKit slaveRepairKit = cluster.Shutdown(slave);
ClearLastTransactionCommitTimestampField(databaseLayout);
HighlyAvailableGraphDatabase repairedSlave = slaveRepairKit.Repair();
cluster.Await(allSeesAllAsAvailable());
assertEquals(LastCommittedTxTimestamp(cluster.Master), LastCommittedTxTimestamp(repairedSlave));
}
finally
{
clusterManager.Stop();
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void lastTxCommitTimestampShouldBeUnknownAfterStartIfNoFiledOrLogsPresent() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void LastTxCommitTimestampShouldBeUnknownAfterStartIfNoFiledOrLogsPresent()
{
ClusterManager clusterManager = (new ClusterManager.Builder(TestDirectory.directory(TestName.MethodName))).withCluster(ClusterManager.clusterOfSize(3)).build();
try
{
clusterManager.Start();
ClusterManager.ManagedCluster cluster = clusterManager.Cluster;
cluster.Await(allSeesAllAsAvailable());
RunSomeTransactions(cluster.Master);
cluster.Sync();
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
DatabaseLayout databaseLayout = slave.DatabaseLayout();
ClusterManager.RepairKit slaveRepairKit = cluster.Shutdown(slave);
ClearLastTransactionCommitTimestampField(databaseLayout);
DeleteLogs(databaseLayout);
HighlyAvailableGraphDatabase repairedSlave = slaveRepairKit.Repair();
cluster.Await(allSeesAllAsAvailable());
assertEquals(Org.Neo4j.Kernel.impl.transaction.log.TransactionIdStore_Fields.UNKNOWN_TX_COMMIT_TIMESTAMP, LastCommittedTxTimestamp(repairedSlave));
}
finally
{
clusterManager.Stop();
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void testFailOver(int clusterSize) throws Throwable
private void TestFailOver(int clusterSize)
{
// given
ClusterManager clusterManager = (new ClusterManager.Builder()).withRootDirectory(Dir.cleanDirectory("failover")).withCluster(ClusterManager.clusterOfSize(clusterSize)).build();
clusterManager.Start();
ClusterManager.ManagedCluster cluster = clusterManager.Cluster;
cluster.Await(ClusterManager.allSeesAllAsAvailable());
HighlyAvailableGraphDatabase oldMaster = cluster.Master;
// When
long start = System.nanoTime();
ClusterManager.RepairKit repairKit = cluster.Fail(oldMaster);
Logger.Logger.warning("Shut down master");
// Then
cluster.Await(ClusterManager.masterAvailable(oldMaster));
long end = System.nanoTime();
Logger.Logger.warning("Failover took:" + (end - start) / 1000000 + "ms");
repairKit.Repair();
Thread.Sleep(3000); // give repaired instance chance to cleanly rejoin and exit faster
clusterManager.SafeShutdown();
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldCopyStoreFromMasterIfBranched() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldCopyStoreFromMasterIfBranched()
{
// GIVEN
File dir = _directory.directory();
ClusterManager clusterManager = _life.add(new ClusterManager.Builder(dir)
.withCluster(clusterOfSize(2)).build());
ClusterManager.ManagedCluster cluster = clusterManager.Cluster;
cluster.Await(allSeesAllAsAvailable());
CreateNode(cluster.Master, "A");
cluster.Sync();
// WHEN
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
File databaseDir = slave.DatabaseLayout().databaseDirectory();
ClusterManager.RepairKit starter = cluster.Shutdown(slave);
HighlyAvailableGraphDatabase master = cluster.Master;
CreateNode(master, "B1");
CreateNode(master, "C");
CreateNodeOffline(databaseDir, "B2");
slave = starter.Repair();
// THEN
cluster.Await(allSeesAllAsAvailable());
slave.BeginTx().close();
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void updatePullerSwitchOnNodeModeSwitch() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void UpdatePullerSwitchOnNodeModeSwitch()
{
ClusterManager.ManagedCluster cluster = ClusterRule.startCluster();
Label firstLabel = Label.label("firstLabel");
CreateLabeledNodeOnMaster(cluster, firstLabel);
// force update puller to work
PullUpdatesOnSlave(cluster);
// node should exist on slave now
CheckLabeledNodeExistenceOnSlave(cluster, firstLabel);
// verify that puller working on slave and not working on master
VerifyUpdatePullerThreads(cluster);
for (int i = 1; i <= 2; i++)
{
// switch roles in cluster - now update puller should be stopped on old slave and start on old master.
ClusterManager.RepairKit repairKit = cluster.Shutdown(cluster.Master);
cluster.Await(masterAvailable());
Label currentLabel = Label.label("label_" + i);
CreateLabeledNodeOnMaster(cluster, currentLabel);
repairKit.Repair();
cluster.Await(allSeesAllAsAvailable(), 120);
// forcing updates pulling
PullUpdatesOnSlave(cluster);
CheckLabeledNodeExistenceOnSlave(cluster, currentLabel);
// checking pulling threads
VerifyUpdatePullerThreads(cluster);
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void newSlaveJoiningClusterShouldNotAcceptOperationsUntilConstraintIsOnline() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void NewSlaveJoiningClusterShouldNotAcceptOperationsUntilConstraintIsOnline()
{
// Given
ClusterManager.ManagedCluster cluster = ClusterRule.startCluster();
string type = type(4);
string key = key(4);
HighlyAvailableGraphDatabase master = cluster.Master;
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
File slaveStoreDirectory = cluster.GetDatabaseDir(slave);
// Crash the slave
ClusterManager.RepairKit shutdownSlave = cluster.Shutdown(slave);
deleteRecursively(slaveStoreDirectory);
using (Transaction tx = master.BeginTx())
{
CreateConstraint(master, type, key);
tx.Success();
}
// When
slave = shutdownSlave.Repair();
// Then
using (Transaction ignored = slave.BeginTx())
{
ConstraintDefinition definition = GetConstraint(slave, type, key);
assertThat(definition, instanceOf(ConstraintDefinitionClass()));
assertThat(single(definition.PropertyKeys), equalTo(key));
ValidateLabelOrRelationshipType(definition, type);
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldHandleSlaveWritingFirstAfterStoryCopy() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldHandleSlaveWritingFirstAfterStoryCopy()
{
// Given
ISet <long> expected = new HashSet <object>();
ClusterManager.ManagedCluster cluster = ClusterRule.startCluster();
HighlyAvailableGraphDatabase master = cluster.Master;
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
// When
expected.Add(CreateOneNode(master));
cluster.Sync();
// ... crash the slave
File slaveStoreDirectory = cluster.GetDatabaseDir(slave);
ClusterManager.RepairKit shutdownSlave = cluster.Shutdown(slave);
deleteRecursively(slaveStoreDirectory);
// ... and slave copy store from master
slave = shutdownSlave.Repair();
// ... and first write after crash occurs on salve
expected.Add(CreateOneNode(slave));
cluster.Sync();
// Then
assertEquals(expected, CollectIds(master));
assertEquals(expected, CollectIds(slave));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void makeSureUpdatePullerGetsGoingAfterMasterSwitch() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void MakeSureUpdatePullerGetsGoingAfterMasterSwitch()
{
ClusterManager.ManagedCluster cluster = ClusterRule.withSharedSetting(HaSettings.pull_interval, PULL_INTERVAL + "ms").startCluster();
cluster.Info("### Creating initial dataset");
long commonNodeId = CreateNodeOnMaster(cluster);
HighlyAvailableGraphDatabase master = cluster.Master;
SetProperty(master, commonNodeId, 1);
cluster.Info("### Initial dataset created");
AwaitPropagation(1, commonNodeId, cluster);
cluster.Info("### Shutting down master");
ClusterManager.RepairKit masterShutdownRK = cluster.Shutdown(master);
cluster.Info("### Awaiting new master");
cluster.Await(masterAvailable(master));
cluster.Await(masterSeesSlavesAsAvailable(1));
cluster.Info("### Doing a write to master");
SetProperty(cluster.Master, commonNodeId, 2);
AwaitPropagation(2, commonNodeId, cluster, master);
cluster.Info("### Repairing cluster");
masterShutdownRK.Repair();
cluster.Await(masterAvailable());
cluster.Await(masterSeesSlavesAsAvailable(2));
cluster.Await(allSeesAllAsAvailable());
cluster.Info("### Awaiting change propagation");
AwaitPropagation(2, commonNodeId, cluster);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void index_objects_can_be_reused_after_role_switch() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void IndexObjectsCanBeReusedAfterRoleSwitch()
{
// GIVEN
// -- an existing index
string key = "key";
string value = "value";
HighlyAvailableGraphDatabase master = Cluster.Master;
long nodeId = CreateNode(master, key, value, true);
Cluster.sync();
// -- get Index and IndexManager references to all dbs
IDictionary <HighlyAvailableGraphDatabase, IndexManager> indexManagers = new Dictionary <HighlyAvailableGraphDatabase, IndexManager>();
IDictionary <HighlyAvailableGraphDatabase, Index <Node> > indexes = new Dictionary <HighlyAvailableGraphDatabase, Index <Node> >();
foreach (HighlyAvailableGraphDatabase db in Cluster.AllMembers)
{
using (Transaction transaction = Db.beginTx())
{
indexManagers[db] = Db.index();
indexes[db] = Db.index().forNodes(key);
transaction.Success();
}
}
// WHEN
// -- there's a master switch
ClusterManager.RepairKit repair = Cluster.shutdown(master);
indexManagers.Remove(master);
indexes.Remove(master);
Cluster.await(ClusterManager.masterAvailable(master));
Cluster.await(ClusterManager.masterSeesSlavesAsAvailable(1));
// THEN
// -- the index instances should still be viable to use
foreach (KeyValuePair <HighlyAvailableGraphDatabase, IndexManager> entry in indexManagers.SetOfKeyValuePairs())
{
HighlyAvailableGraphDatabase db = entry.Key;
using (Transaction transaction = Db.beginTx())
{
IndexManager indexManager = entry.Value;
assertTrue(indexManager.ExistsForNodes(key));
assertEquals(nodeId, indexManager.ForNodes(key).get(key, value).Single.Id);
}
}
foreach (KeyValuePair <HighlyAvailableGraphDatabase, Index <Node> > entry in indexes.SetOfKeyValuePairs())
{
HighlyAvailableGraphDatabase db = entry.Key;
using (Transaction transaction = Db.beginTx())
{
Index <Node> index = entry.Value;
assertEquals(nodeId, index.get(key, value).Single.Id);
}
}
repair.Repair();
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("ResultOfMethodCallIgnored") private org.neo4j.kernel.impl.ha.ClusterManager.RepairKit bringSlaveOfflineAndRemoveStoreFiles(org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster cluster, org.neo4j.kernel.ha.HighlyAvailableGraphDatabase slave) throws java.io.IOException
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
private ClusterManager.RepairKit BringSlaveOfflineAndRemoveStoreFiles(ClusterManager.ManagedCluster cluster, HighlyAvailableGraphDatabase slave)
{
ClusterManager.RepairKit slaveDown = cluster.Shutdown(slave);
File databaseDir = slave.DatabaseLayout().databaseDirectory();
deleteRecursively(databaseDir);
databaseDir.mkdir();
return(slaveDown);
}
private ClusterManager.RepairKit KillIncrementally(ClusterManager.ManagedCluster cluster, HighlyAvailableGraphDatabase failed1, HighlyAvailableGraphDatabase failed2, HighlyAvailableGraphDatabase failed3)
{
ClusterManager.RepairKit firstFailure = cluster.Fail(failed1);
cluster.Await(instanceEvicted(failed1));
cluster.Fail(failed2);
cluster.Await(instanceEvicted(failed2));
cluster.Fail(failed3);
cluster.Await(instanceEvicted(failed3));
return(firstFailure);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void onlineSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterStoreCopy() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void OnlineSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterStoreCopy()
{
/*
* The master has an index that is online.
* Then a slave comes online and contacts the master to get copies of the store files.
* Because the index is online, it should be copied, and the slave should successfully bring the index online.
*/
// GIVEN
ControlledGraphDatabaseFactory dbFactory = new ControlledGraphDatabaseFactory();
ClusterManager.ManagedCluster cluster = ClusterRule.withDbFactory(dbFactory).withSharedSetting(GraphDatabaseSettings.default_schema_provider, _controlledProviderDescriptor.name()).startCluster();
cluster.Await(allSeesAllAsAvailable(), 120);
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
// All slaves in the cluster, except the one I care about, proceed as normal
ProceedAsNormalWithIndexPopulationOnAllSlavesExcept(dbFactory, cluster, slave);
// A slave is offline, and has no store files
ClusterManager.RepairKit slaveDown = BringSlaveOfflineAndRemoveStoreFiles(cluster, slave);
// And I create an index on the master, and wait for population to start
HighlyAvailableGraphDatabase master = cluster.Master;
IDictionary <object, Node> data = CreateSomeData(master);
CreateIndex(master);
dbFactory.AwaitPopulationStarted(master);
// And the population finishes
dbFactory.TriggerFinish(master);
IndexDefinition index;
using (Transaction tx = master.BeginTx())
{
index = single(master.Schema().Indexes);
AwaitIndexOnline(index, master, data);
tx.Success();
}
// WHEN the slave comes online after population has finished on the master
slave = slaveDown.Repair();
cluster.Await(allSeesAllAsAvailable());
cluster.Sync();
// THEN the index should work on the slave
dbFactory.TriggerFinish(slave);
using (Transaction tx = slave.BeginTx())
{
AwaitIndexOnline(index, slave, data);
tx.Success();
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldContinueServingBoltRequestsBetweenInternalRestarts() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldContinueServingBoltRequestsBetweenInternalRestarts()
{
// given
/*
* Interestingly, it is enough to simply start a slave and then direct sessions to it. The problem seems
* to arise immediately, since simply from startup to being into SLAVE at least one internal restart happens
* and that seems sufficient to break the bolt server.
* However, that would make the test really weird, so we'll start the cluster, make sure we can connect and
* then isolate the slave, make it shutdown internally, then have it rejoin and it will switch to slave.
* At the end of this process, it must still be possible to open and execute transactions against the instance.
*/
ClusterManager.ManagedCluster cluster = ClusterRule.startCluster();
HighlyAvailableGraphDatabase slave1 = cluster.AnySlave;
Driver driver = GraphDatabase.driver(cluster.GetBoltAddress(slave1), AuthTokens.basic("neo4j", "neo4j"));
/*
* We'll use a bookmark to enforce use of kernel internals by the bolt server, to make sure that parts that are
* switched during an internal restart are actually refreshed. Technically, this is not necessary, since the
* bolt server makes such use for every request. But this puts a nice bow on top of it.
*/
string lastBookmark = InExpirableSession(driver, Driver.session, s =>
{
using (Transaction tx = s.beginTransaction())
{
tx.run("CREATE (person:Person {name: {name}, title: {title}})", parameters("name", "Webber", "title", "Mr"));
tx.success();
}
return(s.lastBookmark());
});
// when
ClusterManager.RepairKit slaveFailRK = cluster.Fail(slave1);
cluster.Await(entireClusterSeesMemberAsNotAvailable(slave1));
slaveFailRK.Repair();
cluster.Await(masterSeesMembers(3));
// then
int?count = InExpirableSession(driver, Driver.session, s =>
{
Record record;
using (Transaction tx = s.beginTransaction(lastBookmark))
{
record = tx.run("MATCH (n:Person) RETURN COUNT(*) AS count").next();
tx.success();
}
return(record.get("count").asInt());
});
assertEquals(1, count.Value);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void reelectTheSameMasterMakingItGoToPendingAndBack(org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster cluster) throws Throwable
private void ReelectTheSameMasterMakingItGoToPendingAndBack(ClusterManager.ManagedCluster cluster)
{
HighlyAvailableGraphDatabase master = cluster.Master;
// Fail master and wait for master to go to pending, since it detects it's partitioned away
ClusterManager.RepairKit masterRepair = cluster.Fail(master, false, ClusterManager.NetworkFlag.IN, ClusterManager.NetworkFlag.OUT);
cluster.Await(memberThinksItIsRole(master, UNKNOWN));
// Then Immediately repair
masterRepair.Repair();
// Wait for this instance to go to master again, since the other instances are slave only
cluster.Await(memberThinksItIsRole(master, MASTER));
cluster.Await(ClusterManager.masterAvailable());
assertEquals(master, cluster.Master);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void slaveMustConnectLockManagerToNewMasterAfterTwoOtherClusterMembersRoleSwitch() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void SlaveMustConnectLockManagerToNewMasterAfterTwoOtherClusterMembersRoleSwitch()
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.neo4j.kernel.ha.HighlyAvailableGraphDatabase initialMaster = cluster.getMaster();
HighlyAvailableGraphDatabase initialMaster = _cluster.Master;
HighlyAvailableGraphDatabase firstSlave = _cluster.AnySlave;
HighlyAvailableGraphDatabase secondSlave = _cluster.getAnySlave(firstSlave);
// Run a transaction on the slaves, to make sure that a master connection has been initialised in all
// internal pools.
using (Transaction tx = firstSlave.BeginTx())
{
firstSlave.CreateNode();
tx.Success();
}
using (Transaction tx = secondSlave.BeginTx())
{
secondSlave.CreateNode();
tx.Success();
}
_cluster.sync();
ClusterManager.RepairKit failedMaster = _cluster.fail(initialMaster);
_cluster.await(ClusterManager.masterAvailable(initialMaster));
failedMaster.Repair();
_cluster.await(ClusterManager.masterAvailable(initialMaster));
_cluster.await(ClusterManager.allSeesAllAsAvailable());
// The cluster has now switched the master role to one of the slaves.
// The slave that didn't switch, should still have done the work to reestablish the connection to the new
// master.
HighlyAvailableGraphDatabase slave = _cluster.getAnySlave(initialMaster);
using (Transaction tx = slave.BeginTx())
{
slave.CreateNode();
tx.Success();
}
// We assert that the transaction above does not throw any exceptions, and that we have now created 3 nodes.
HighlyAvailableGraphDatabase master = _cluster.Master;
using (Transaction tx = master.BeginTx())
{
assertThat(Iterables.count(master.AllNodes), @is(3L));
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test(timeout = TIMEOUT_MILLIS) public void lockCleanupOnModeSwitch() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void LockCleanupOnModeSwitch()
{
HighlyAvailableGraphDatabase master = _cluster.Master;
CreateNodeOnMaster(_testLabel, master);
HighlyAvailableGraphDatabase slave = _cluster.AnySlave;
ClusterManager.RepairKit repairKit = TakeExclusiveLockAndKillSlave(_testLabel, slave);
// repair of slave and new mode switch cycle on all members
repairKit.Repair();
_cluster.await(allSeesAllAsAvailable());
HighlyAvailableGraphDatabase clusterMaster = _cluster.Master;
// now it should be possible to take exclusive lock on the same node
TakeExclusiveLockOnSameNodeAfterSwitch(_testLabel, master, clusterMaster);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void testFailoverWithAdditionalSlave(int clusterSize, int[] slaveIndexes) throws Throwable
private void TestFailoverWithAdditionalSlave( int clusterSize, int[] slaveIndexes )
{
File root = Dir.cleanDirectory( "testcluster_" + Name.MethodName );
ClusterManager manager = ( new ClusterManager.Builder() ).withRootDirectory(root).withCluster(ClusterManager.clusterOfSize(clusterSize)).build();
try
{
manager.Start();
ClusterManager.ManagedCluster cluster = manager.Cluster;
cluster.Await( allSeesAllAsAvailable() );
cluster.Await( masterAvailable() );
ICollection<HighlyAvailableGraphDatabase> failed = new List<HighlyAvailableGraphDatabase>();
ICollection<ClusterManager.RepairKit> repairKits = new List<ClusterManager.RepairKit>();
foreach ( int slaveIndex in slaveIndexes )
{
HighlyAvailableGraphDatabase nthSlave = GetNthSlave( cluster, slaveIndex );
failed.Add( nthSlave );
ClusterManager.RepairKit repairKit = cluster.Fail( nthSlave );
repairKits.Add( repairKit );
}
HighlyAvailableGraphDatabase oldMaster = cluster.Master;
failed.Add( oldMaster );
repairKits.Add( cluster.Fail( oldMaster ) );
cluster.Await( masterAvailable( ToArray( failed ) ) );
foreach ( ClusterManager.RepairKit repairKit in repairKits )
{
repairKit.Repair();
}
Thread.Sleep( 3000 ); // give repaired instances a chance to cleanly rejoin and exit faster
}
finally
{
manager.SafeShutdown();
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void masterRejoinsAfterFailureAndReelection() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void MasterRejoinsAfterFailureAndReelection()
{
// Given
HighlyAvailableGraphDatabase initialMaster = _cluster.Master;
// When
_cluster.info("Fail master");
ClusterManager.RepairKit kit = _cluster.fail(initialMaster);
_cluster.info("Wait for 2 to become master and 3 slave");
_cluster.await(masterAvailable(initialMaster));
_cluster.await(masterSeesSlavesAsAvailable(1));
_cluster.info("Repair 1");
kit.Repair();
// Then
_cluster.info("Wait for cluster recovery");
_cluster.await(masterAvailable());
_cluster.await(allSeesAllAsAvailable());
assertEquals(3, _cluster.size());
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void isolatedMasterShouldRemoveSelfFromClusterAndBecomeReadOnly() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void IsolatedMasterShouldRemoveSelfFromClusterAndBecomeReadOnly()
{
int clusterSize = 3;
ClusterManager manager = (new ClusterManager.Builder()).withRootDirectory(Dir.cleanDirectory("testcluster")).withCluster(ClusterManager.clusterOfSize(clusterSize)).build();
try
{
manager.Start();
ClusterManager.ManagedCluster cluster = manager.Cluster;
cluster.Await(allSeesAllAsAvailable());
cluster.Await(masterAvailable());
HighlyAvailableGraphDatabase oldMaster = cluster.Master;
System.Threading.CountdownEvent masterTransitionLatch = new System.Threading.CountdownEvent(1);
SetupForWaitOnSwitchToDetached(oldMaster, masterTransitionLatch);
AddSomeData(oldMaster);
ClusterManager.RepairKit fail = cluster.fail(oldMaster, Enum.GetValues(typeof(ClusterManager.NetworkFlag)));
cluster.Await(instanceEvicted(oldMaster), 20);
masterTransitionLatch.await();
EnsureInstanceIsReadOnlyInPendingState(oldMaster);
fail.Repair();
cluster.Await(allSeesAllAsAvailable());
EnsureInstanceIsWritable(oldMaster);
}
finally
{
manager.SafeShutdown();
}
}
/// <summary>
/// Main difference to <seealso cref="shouldCopyStoreFromMasterIfBranched()"/> is that no instances are shut down
/// during the course of the test. This to test functionality of some internal components being restarted.
/// </summary>
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("unchecked") @Test public void shouldCopyStoreFromMasterIfBranchedInLiveScenario() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldCopyStoreFromMasterIfBranchedInLiveScenario()
{
// GIVEN a cluster of 3, all having the same data (node A)
// thor is whoever is the master to begin with
// odin is whoever is picked as _the_ slave given thor as initial master
File storeDirectory = _directory.directory();
ClusterManager clusterManager = _life.add(new ClusterManager.Builder(storeDirectory)
.withSharedConfig(stringMap(HaSettings.tx_push_factor.name(), "0", HaSettings.pull_interval.name(), "0")).build());
ClusterManager.ManagedCluster cluster = clusterManager.Cluster;
cluster.Await(allSeesAllAsAvailable());
HighlyAvailableGraphDatabase thor = cluster.Master;
string indexName = "valhalla";
CreateNode(thor, "A", AndIndexInto(indexName));
cluster.Sync();
// WHEN creating a node B1 on thor (note the disabled cluster transaction propagation)
CreateNode(thor, "B1", AndIndexInto(indexName));
// and right after that failing the master so that it falls out of the cluster
HighlyAvailableGraphDatabase odin = cluster.AnySlave;
cluster.Info(format("%n ==== TAMPERING WITH " + thor + "'s CABLES ====%n"));
ClusterManager.RepairKit thorRepairKit = cluster.Fail(thor);
// try to create a transaction on odin until it succeeds
cluster.Await(ClusterManager.masterAvailable(thor));
cluster.Await(ClusterManager.memberThinksItIsRole(odin, HighAvailabilityModeSwitcher.MASTER));
assertTrue(odin.Master);
RetryOnTransactionFailure(odin, db => createNode(db, "B2", AndIndexInto(indexName)));
// perform transactions so that index files changes under the hood
ISet <File> odinLuceneFilesBefore = Iterables.asSet(GatherLuceneFiles(odin, indexName));
for (char prefix = 'C'; !Changed(odinLuceneFilesBefore, Iterables.asSet(GatherLuceneFiles(odin, indexName))); prefix++)
{
char fixedPrefix = prefix;
RetryOnTransactionFailure(odin, db => createNodes(odin, fixedPrefix.ToString(), 10_000, AndIndexInto(indexName)));
cluster.Force(); // Force will most likely cause lucene explicit indexes to commit and change file structure
}
// so anyways, when thor comes back into the cluster
cluster.Info(format("%n ==== REPAIRING CABLES ====%n"));
cluster.Await(memberThinksItIsRole(thor, UNKNOWN));
BranchMonitor thorHasBranched = InstallBranchedDataMonitor(cluster.GetMonitorsByDatabase(thor));
thorRepairKit.Repair();
cluster.Await(memberThinksItIsRole(thor, SLAVE));
cluster.Await(memberThinksItIsRole(odin, MASTER));
cluster.Await(allSeesAllAsAvailable());
assertFalse(thor.Master);
assertTrue("No store-copy performed", thorHasBranched.CopyCompleted);
assertTrue("Store-copy unsuccessful", thorHasBranched.CopySuccessful);
// Now do some more transactions on current master (odin) and have thor pull those
for (int i = 0; i < 3; i++)
{
int ii = i;
RetryOnTransactionFailure(odin, db => createNodes(odin, ("" + ii).ToString(), 10, AndIndexInto(indexName)));
cluster.Sync();
cluster.Force();
}
// THEN thor should be a slave, having copied a store from master and good to go
assertFalse(HasNode(thor, "B1"));
assertTrue(HasNode(thor, "B2"));
assertTrue(HasNode(thor, "C-0"));
assertTrue(HasNode(thor, "0-0"));
assertTrue(HasNode(odin, "0-0"));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test @Ignore public void slaveShouldServeTxsAfterMasterLostQuorumWentToPendingAndThenQuorumWasRestored() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void SlaveShouldServeTxsAfterMasterLostQuorumWentToPendingAndThenQuorumWasRestored()
{
// GIVEN: cluster with 3 members
HighlyAvailableGraphDatabase master = _cluster.Master;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final HighlyAvailableGraphDatabase slave1 = cluster.getAnySlave();
HighlyAvailableGraphDatabase slave1 = _cluster.AnySlave;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final HighlyAvailableGraphDatabase slave2 = cluster.getAnySlave(slave1);
HighlyAvailableGraphDatabase slave2 = _cluster.getAnySlave(slave1);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch slave1Left = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent slave1Left = new System.Threading.CountdownEvent(1);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch slave2Left = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent slave2Left = new System.Threading.CountdownEvent(1);
ClusterClientOf(master).addHeartbeatListener(new HeartbeatListener_AdapterAnonymousInnerClass(this, slave1, slave2, slave1Left, slave2Left));
// fail slave1 and await master to spot the failure
ClusterManager.RepairKit slave1RepairKit = _cluster.fail(slave1);
assertTrue(slave1Left.await(60, SECONDS));
// fail slave2 and await master to spot the failure
ClusterManager.RepairKit slave2RepairKit = _cluster.fail(slave2);
assertTrue(slave2Left.await(60, SECONDS));
// master loses quorum and goes to PENDING, cluster is unavailable
_cluster.await(masterAvailable().negate());
assertEquals(HighAvailabilityMemberState.PENDING, master.InstanceState);
// WHEN: both slaves are repaired, majority restored, quorum can be achieved
slave1RepairKit.Repair();
slave2RepairKit.Repair();
// whole cluster looks fine, but slaves have stale value of the epoch if they rejoin the cluster in SLAVE state
_cluster.await(masterAvailable());
_cluster.await(masterSeesSlavesAsAvailable(2));
HighlyAvailableGraphDatabase newMaster = _cluster.Master;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final HighlyAvailableGraphDatabase newSlave1 = cluster.getAnySlave();
HighlyAvailableGraphDatabase newSlave1 = _cluster.AnySlave;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final HighlyAvailableGraphDatabase newSlave2 = cluster.getAnySlave(newSlave1);
HighlyAvailableGraphDatabase newSlave2 = _cluster.getAnySlave(newSlave1);
// now adding another failing listener and wait for the failure due to stale epoch
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch slave1Unavailable = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent slave1Unavailable = new System.Threading.CountdownEvent(1);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch slave2Unavailable = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent slave2Unavailable = new System.Threading.CountdownEvent(1);
ClusterMemberEvents clusterEvents = newMaster.DependencyResolver.resolveDependency(typeof(ClusterMemberEvents));
clusterEvents.AddClusterMemberListener(new ClusterMemberListener_AdapterAnonymousInnerClass(this, newSlave1, newSlave2, slave1Unavailable, slave2Unavailable));
// attempt to perform transactions on both slaves throws, election is triggered
AttemptTransactions(newSlave1, newSlave2);
// set a timeout in case the instance does not have stale epoch
assertTrue(slave1Unavailable.await(60, TimeUnit.SECONDS));
assertTrue(slave2Unavailable.await(60, TimeUnit.SECONDS));
// THEN: done with election, cluster feels good and able to serve transactions
_cluster.info("Waiting for cluster to stabilize");
_cluster.await(allSeesAllAsAvailable());
_cluster.info("Assert ok");
assertNotNull(CreateNodeOn(newMaster));
assertNotNull(CreateNodeOn(newSlave1));
assertNotNull(CreateNodeOn(newSlave2));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void populatingSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterStoreCopy() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void PopulatingSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterStoreCopy()
{
/*
* The master has an index that is currently populating.
* Then a slave comes online and contacts the master to get copies of the store files.
* Because the index is still populating, it won't be copied. Instead the slave will build its own.
* We want to observe that the slave builds an index that eventually comes online.
*/
// GIVEN
ControlledGraphDatabaseFactory dbFactory = new ControlledGraphDatabaseFactory(_isMaster);
ClusterManager.ManagedCluster cluster = ClusterRule.withDbFactory(dbFactory).withSharedSetting(GraphDatabaseSettings.default_schema_provider, NativeLuceneFusionIndexProviderFactory20.DESCRIPTOR.name()).startCluster();
try
{
cluster.Await(allSeesAllAsAvailable());
HighlyAvailableGraphDatabase slave = cluster.AnySlave;
// A slave is offline, and has no store files
ClusterManager.RepairKit slaveDown = BringSlaveOfflineAndRemoveStoreFiles(cluster, slave);
// And I create an index on the master, and wait for population to start
HighlyAvailableGraphDatabase master = cluster.Master;
IDictionary <object, Node> data = CreateSomeData(master);
CreateIndex(master);
dbFactory.AwaitPopulationStarted(master);
// WHEN the slave comes online before population has finished on the master
slave = slaveDown.Repair();
cluster.Await(allSeesAllAsAvailable(), 180);
cluster.Sync();
// THEN, population should finish successfully on both master and slave
dbFactory.TriggerFinish(master);
// Check master
IndexDefinition index;
using (Transaction tx = master.BeginTx())
{
index = single(master.Schema().Indexes);
AwaitIndexOnline(index, master, data);
tx.Success();
}
// Check slave
using (Transaction tx = slave.BeginTx())
{
AwaitIndexOnline(index, slave, data);
tx.Success();
}
}
finally
{
foreach (HighlyAvailableGraphDatabase db in cluster.AllMembers)
{
dbFactory.TriggerFinish(db);
}
}
}
