public virtual void TestBackupNodeTailsEdits()
{
Configuration conf = new HdfsConfiguration();
HAUtil.SetAllowStandbyReads(conf, true);
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
BackupNode backup = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
fileSys = cluster.GetFileSystem();
backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1);
BackupImage bnImage = (BackupImage)backup.GetFSImage();
TestBNInSync(cluster, backup, 1);
// Force a roll -- BN should roll with NN.
NameNode nn = cluster.GetNameNode();
NamenodeProtocols nnRpc = nn.GetRpcServer();
nnRpc.RollEditLog();
NUnit.Framework.Assert.AreEqual(bnImage.GetEditLog().GetCurSegmentTxId(), nn.GetFSImage
().GetEditLog().GetCurSegmentTxId());
// BN should stay in sync after roll
TestBNInSync(cluster, backup, 2);
long nnImageBefore = nn.GetFSImage().GetStorage().GetMostRecentCheckpointTxId();
// BN checkpoint
backup.DoCheckpoint();
// NN should have received a new image
long nnImageAfter = nn.GetFSImage().GetStorage().GetMostRecentCheckpointTxId();
NUnit.Framework.Assert.IsTrue("nn should have received new checkpoint. before: "
+ nnImageBefore + " after: " + nnImageAfter, nnImageAfter > nnImageBefore);
// BN should stay in sync after checkpoint
TestBNInSync(cluster, backup, 3);
// Stop BN
Storage.StorageDirectory sd = bnImage.GetStorage().GetStorageDir(0);
backup.Stop();
backup = null;
// When shutting down the BN, it shouldn't finalize logs that are
// still open on the NN
FileJournalManager.EditLogFile editsLog = FSImageTestUtil.FindLatestEditsLog(sd);
NUnit.Framework.Assert.AreEqual(editsLog.GetFirstTxId(), nn.GetFSImage().GetEditLog
().GetCurSegmentTxId());
NUnit.Framework.Assert.IsTrue("Should not have finalized " + editsLog, editsLog.IsInProgress
());
// do some edits
NUnit.Framework.Assert.IsTrue(fileSys.Mkdirs(new Path("/edit-while-bn-down")));
// start a new backup node
backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1);
TestBNInSync(cluster, backup, 4);
NUnit.Framework.Assert.IsNotNull(backup.GetNamesystem().GetFileInfo("/edit-while-bn-down"
, false));
}
finally
{
Log.Info("Shutting down...");
if (backup != null)
{
backup.Stop();
}
if (fileSys != null)
{
fileSys.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
AssertStorageDirsMatch(cluster.GetNameNode(), backup);
}
public virtual void PurgeLog(FileJournalManager.EditLogFile log)
{
Log.Info("Purging old edit log " + log);
DeleteOrWarn(log.GetFile());
}
public virtual void TestSaveNamespace()
{
DistributedFileSystem fs = null;
try
{
Configuration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
string renewer = UserGroupInformation.GetLoginUser().GetUserName();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = namesystem
.GetDelegationToken(new Text(renewer));
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = namesystem
.GetDelegationToken(new Text(renewer));
// Saving image without safe mode should fail
DFSAdmin admin = new DFSAdmin(conf);
string[] args = new string[] { "-saveNamespace" };
// verify that the edits file is NOT empty
NameNode nn = cluster.GetNameNode();
foreach (Storage.StorageDirectory sd in nn.GetFSImage().GetStorage().DirIterable(
null))
{
FileJournalManager.EditLogFile log = FSImageTestUtil.FindLatestEditsLog(sd);
NUnit.Framework.Assert.IsTrue(log.IsInProgress());
log.ValidateLog();
long numTransactions = (log.GetLastTxId() - log.GetFirstTxId()) + 1;
NUnit.Framework.Assert.AreEqual("In-progress log " + log + " should have 5 transactions"
, 5, numTransactions);
}
// Saving image in safe mode should succeed
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
try
{
admin.Run(args);
}
catch (Exception e)
{
throw new IOException(e.Message);
}
// verify that the edits file is empty except for the START txn
foreach (Storage.StorageDirectory sd_1 in nn.GetFSImage().GetStorage().DirIterable
(null))
{
FileJournalManager.EditLogFile log = FSImageTestUtil.FindLatestEditsLog(sd_1);
NUnit.Framework.Assert.IsTrue(log.IsInProgress());
log.ValidateLog();
long numTransactions = (log.GetLastTxId() - log.GetFirstTxId()) + 1;
NUnit.Framework.Assert.AreEqual("In-progress log " + log + " should only have START txn"
, 1, numTransactions);
}
// restart cluster
cluster.Shutdown();
cluster = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false
).Build();
cluster.WaitActive();
//Should be able to renew & cancel the delegation token after cluster restart
try
{
RenewToken(token1);
RenewToken(token2);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Could not renew or cancel the token");
}
namesystem = cluster.GetNamesystem();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token3 = namesystem
.GetDelegationToken(new Text(renewer));
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token4 = namesystem
.GetDelegationToken(new Text(renewer));
// restart cluster again
cluster.Shutdown();
cluster = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false
).Build();
cluster.WaitActive();
namesystem = cluster.GetNamesystem();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token5 = namesystem
.GetDelegationToken(new Text(renewer));
try
{
RenewToken(token1);
RenewToken(token2);
RenewToken(token3);
RenewToken(token4);
RenewToken(token5);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Could not renew or cancel the token");
}
// restart cluster again
cluster.Shutdown();
cluster = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false
).Build();
cluster.WaitActive();
namesystem = cluster.GetNamesystem();
try
{
RenewToken(token1);
CancelToken(token1);
RenewToken(token2);
CancelToken(token2);
RenewToken(token3);
CancelToken(token3);
RenewToken(token4);
CancelToken(token4);
RenewToken(token5);
CancelToken(token5);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Could not renew or cancel the token");
}
}
finally
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}