/// <exception cref="System.Exception"/>
public static void WaitForCheckpoint(MiniDFSCluster cluster, int nnIdx, IList <int
> txids)
{
long start = Time.Now();
while (true)
{
try
{
FSImageTestUtil.AssertNNHasCheckpoints(cluster, nnIdx, txids);
return;
}
catch (Exception err)
{
if (Time.Now() - start > 10000)
{
throw;
}
else
{
Sharpen.Thread.Sleep(300);
}
}
}
}
/// <exception cref="System.Exception"/>
private void TestUpgrade(TestBootstrapStandbyWithQJM.UpgradeState state)
{
cluster.TransitionToActive(0);
Configuration confNN1 = cluster.GetConfiguration(1);
FilePath current = cluster.GetNameNode(1).GetFSImage().GetStorage().GetStorageDir
(0).GetCurrentDir();
FilePath tmp = cluster.GetNameNode(1).GetFSImage().GetStorage().GetStorageDir(0).
GetPreviousTmp();
// shut down nn1
cluster.ShutdownNameNode(1);
// make NN0 in upgrade state
FSImage fsImage0 = cluster.GetNameNode(0).GetNamesystem().GetFSImage();
Whitebox.SetInternalState(fsImage0, "isUpgradeFinalized", false);
switch (state)
{
case TestBootstrapStandbyWithQJM.UpgradeState.Recover:
{
// rename the current directory to previous.tmp in nn1
NNStorage.Rename(current, tmp);
break;
}
case TestBootstrapStandbyWithQJM.UpgradeState.Format:
{
// rename the current directory to a random name so it's not formatted
FilePath wrongPath = new FilePath(current.GetParentFile(), "wrong");
NNStorage.Rename(current, wrongPath);
break;
}
default:
{
break;
}
}
int rc = BootstrapStandby.Run(new string[] { "-force" }, confNN1);
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of(0));
FSImageTestUtil.AssertNNFilesMatch(cluster);
// make sure the NN1 is in upgrade state, i.e., the previous directory has
// been successfully created
cluster.RestartNameNode(1);
NUnit.Framework.Assert.IsFalse(cluster.GetNameNode(1).GetNamesystem().IsUpgradeFinalized
());
}
public virtual void TestBootstrapStandbyWithActiveNN()
{
// make the first NN in active state
cluster.TransitionToActive(0);
Configuration confNN1 = cluster.GetConfiguration(1);
// shut down nn1
cluster.ShutdownNameNode(1);
int rc = BootstrapStandby.Run(new string[] { "-force" }, confNN1);
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the standby
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of(0));
FSImageTestUtil.AssertNNFilesMatch(cluster);
}
public virtual void TestBootstrapStandbyWithActiveNN()
{
// make nn0 active
cluster.TransitionToActive(0);
// do ops and generate in-progress edit log data
Configuration confNN1 = cluster.GetConfiguration(1);
DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs
(cluster, confNN1);
for (int i = 1; i <= 10; i++)
{
dfs.Mkdirs(new Path("/test" + i));
}
dfs.Close();
// shutdown nn1 and delete its edit log files
cluster.ShutdownNameNode(1);
DeleteEditLogIfExists(confNN1);
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
true);
cluster.GetNameNodeRpc(0).SaveNamespace();
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
true);
// check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
// immediately after saveNamespace
int rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive" }, confNN1
);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 6, rc);
// check with -skipSharedEditsCheck
rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive", "-skipSharedEditsCheck" }, confNN1);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 0, rc);
// Checkpoint as fast as we can, in a tight loop.
confNN1.SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, 1);
cluster.RestartNameNode(1);
cluster.TransitionToStandby(1);
NameNode nn0 = cluster.GetNameNode(0);
HATestUtil.WaitForStandbyToCatchUp(nn0, cluster.GetNameNode(1));
long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId
();
HATestUtil.WaitForCheckpoint(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
// Should have copied over the namespace
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
FSImageTestUtil.AssertNNFilesMatch(cluster);
}
public virtual void TestSuccessfulBaseCase()
{
RemoveStandbyNameDirs();
try
{
cluster.RestartNameNode(1);
NUnit.Framework.Assert.Fail("Did not throw");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("storage directory does not exist or is not accessible"
, ioe);
}
int rc = BootstrapStandby.Run(new string[] { "-nonInteractive" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of(0));
FSImageTestUtil.AssertNNFilesMatch(cluster);
// We should now be able to start the standby successfully.
cluster.RestartNameNode(1);
}
public virtual void TestDownloadingLaterCheckpoint()
{
// Roll edit logs a few times to inflate txid
nn0.GetRpcServer().RollEditLog();
nn0.GetRpcServer().RollEditLog();
// Make checkpoint
NameNodeAdapter.EnterSafeMode(nn0, false);
NameNodeAdapter.SaveNamespace(nn0);
NameNodeAdapter.LeaveSafeMode(nn0);
long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId
();
NUnit.Framework.Assert.AreEqual(6, expectedCheckpointTxId);
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
FSImageTestUtil.AssertNNFilesMatch(cluster);
// We should now be able to start the standby successfully.
cluster.RestartNameNode(1);
}