public virtual void TestBackupNodeTailsEdits() { Configuration conf = new HdfsConfiguration(); HAUtil.SetAllowStandbyReads(conf, true); MiniDFSCluster cluster = null; FileSystem fileSys = null; BackupNode backup = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); fileSys = cluster.GetFileSystem(); backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1); BackupImage bnImage = (BackupImage)backup.GetFSImage(); TestBNInSync(cluster, backup, 1); // Force a roll -- BN should roll with NN. NameNode nn = cluster.GetNameNode(); NamenodeProtocols nnRpc = nn.GetRpcServer(); nnRpc.RollEditLog(); NUnit.Framework.Assert.AreEqual(bnImage.GetEditLog().GetCurSegmentTxId(), nn.GetFSImage ().GetEditLog().GetCurSegmentTxId()); // BN should stay in sync after roll TestBNInSync(cluster, backup, 2); long nnImageBefore = nn.GetFSImage().GetStorage().GetMostRecentCheckpointTxId(); // BN checkpoint backup.DoCheckpoint(); // NN should have received a new image long nnImageAfter = nn.GetFSImage().GetStorage().GetMostRecentCheckpointTxId(); NUnit.Framework.Assert.IsTrue("nn should have received new checkpoint. before: " + nnImageBefore + " after: " + nnImageAfter, nnImageAfter > nnImageBefore); // BN should stay in sync after checkpoint TestBNInSync(cluster, backup, 3); // Stop BN Storage.StorageDirectory sd = bnImage.GetStorage().GetStorageDir(0); backup.Stop(); backup = null; // When shutting down the BN, it shouldn't finalize logs that are // still open on the NN FileJournalManager.EditLogFile editsLog = FSImageTestUtil.FindLatestEditsLog(sd); NUnit.Framework.Assert.AreEqual(editsLog.GetFirstTxId(), nn.GetFSImage().GetEditLog ().GetCurSegmentTxId()); NUnit.Framework.Assert.IsTrue("Should not have finalized " + editsLog, editsLog.IsInProgress ()); // do some edits NUnit.Framework.Assert.IsTrue(fileSys.Mkdirs(new Path("/edit-while-bn-down"))); // start a new backup node backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1); TestBNInSync(cluster, backup, 4); NUnit.Framework.Assert.IsNotNull(backup.GetNamesystem().GetFileInfo("/edit-while-bn-down" , false)); } finally { Log.Info("Shutting down..."); if (backup != null) { backup.Stop(); } if (fileSys != null) { fileSys.Close(); } if (cluster != null) { cluster.Shutdown(); } } AssertStorageDirsMatch(cluster.GetNameNode(), backup); }
public virtual void PurgeLog(FileJournalManager.EditLogFile log) { Log.Info("Purging old edit log " + log); DeleteOrWarn(log.GetFile()); }
public virtual void TestSaveNamespace() { DistributedFileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); string renewer = UserGroupInformation.GetLoginUser().GetUserName(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = namesystem .GetDelegationToken(new Text(renewer)); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = namesystem .GetDelegationToken(new Text(renewer)); // Saving image without safe mode should fail DFSAdmin admin = new DFSAdmin(conf); string[] args = new string[] { "-saveNamespace" }; // verify that the edits file is NOT empty NameNode nn = cluster.GetNameNode(); foreach (Storage.StorageDirectory sd in nn.GetFSImage().GetStorage().DirIterable( null)) { FileJournalManager.EditLogFile log = FSImageTestUtil.FindLatestEditsLog(sd); NUnit.Framework.Assert.IsTrue(log.IsInProgress()); log.ValidateLog(); long numTransactions = (log.GetLastTxId() - log.GetFirstTxId()) + 1; NUnit.Framework.Assert.AreEqual("In-progress log " + log + " should have 5 transactions" , 5, numTransactions); } // Saving image in safe mode should succeed fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); try { admin.Run(args); } catch (Exception e) { throw new IOException(e.Message); } // verify that the edits file is empty except for the START txn foreach (Storage.StorageDirectory sd_1 in nn.GetFSImage().GetStorage().DirIterable (null)) { FileJournalManager.EditLogFile log = FSImageTestUtil.FindLatestEditsLog(sd_1); NUnit.Framework.Assert.IsTrue(log.IsInProgress()); log.ValidateLog(); long numTransactions = (log.GetLastTxId() - log.GetFirstTxId()) + 1; NUnit.Framework.Assert.AreEqual("In-progress log " + log + " should only have START txn" , 1, numTransactions); } // restart cluster cluster.Shutdown(); cluster = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false ).Build(); cluster.WaitActive(); //Should be able to renew & cancel the delegation token after cluster restart try { RenewToken(token1); RenewToken(token2); } catch (IOException) { NUnit.Framework.Assert.Fail("Could not renew or cancel the token"); } namesystem = cluster.GetNamesystem(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token3 = namesystem .GetDelegationToken(new Text(renewer)); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token4 = namesystem .GetDelegationToken(new Text(renewer)); // restart cluster again cluster.Shutdown(); cluster = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false ).Build(); cluster.WaitActive(); namesystem = cluster.GetNamesystem(); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token5 = namesystem .GetDelegationToken(new Text(renewer)); try { RenewToken(token1); RenewToken(token2); RenewToken(token3); RenewToken(token4); RenewToken(token5); } catch (IOException) { NUnit.Framework.Assert.Fail("Could not renew or cancel the token"); } // restart cluster again cluster.Shutdown(); cluster = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Format(false ).Build(); cluster.WaitActive(); namesystem = cluster.GetNamesystem(); try { RenewToken(token1); CancelToken(token1); RenewToken(token2); CancelToken(token2); RenewToken(token3); CancelToken(token3); RenewToken(token4); CancelToken(token4); RenewToken(token5); CancelToken(token5); } catch (IOException) { NUnit.Framework.Assert.Fail("Could not renew or cancel the token"); } } finally { if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }