/// <exception cref="System.Exception"/> public virtual void TestQuery() { Configuration conf = new Configuration(); MiniQJMHACluster cluster = null; try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfsCluster.ShutdownNameNode(1); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Query); NUnit.Framework.Assert.IsFalse(info.CreatedRollbackImages()); dfsCluster.RestartNameNode(1); QueryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. NUnit.Framework.Assert.IsTrue(dfsCluster.GetNamesystem(0).GetFSImage().HasRollbackFSImage ()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
// expected because the old active will be unable to flush the // end-of-segment op since it is fenced /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/> public virtual void TestReadEventsWithTimeout() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).Build(); try { cluster.GetDfsCluster().WaitActive(); cluster.GetDfsCluster().TransitionToActive(0); DFSClient client = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress (), conf); DFSInotifyEventInputStream eis = client.GetInotifyEventStream(); ScheduledExecutorService ex = Executors.NewSingleThreadScheduledExecutor(); ex.Schedule(new _Runnable_463(client), 1, TimeUnit.Seconds); // test will fail // a very generous wait period -- the edit will definitely have been // processed by the time this is up EventBatch batch = eis.Poll(5, TimeUnit.Seconds); NUnit.Framework.Assert.IsNotNull(batch); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); NUnit.Framework.Assert.AreEqual("/dir", ((Event.CreateEvent)batch.GetEvents()[0]) .GetPath()); } finally { cluster.Shutdown(); } }
public virtual void SetUpCluster() { conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointCheckPeriodKey, 1); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 1); conf.SetInt(DFSConfigKeys.DfsNamenodeNumCheckpointsRetainedKey, 10); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); HAUtil.SetAllowStandbyReads(conf, true); if (clusterType == TestFailureToReadEdits.TestType.SharedDirHa) { MiniDFSNNTopology topology = MiniQJMHACluster.CreateDefaultTopology(10000); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).CheckExitOnShutdown (false).Build(); } else { MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(0).CheckExitOnShutdown(false); miniQjmHaCluster = builder.Build(); cluster = miniQjmHaCluster.GetDfsCluster(); } cluster.WaitActive(); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); cluster.TransitionToActive(0); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); }
public virtual void TestFinalizeWithJournalNodes() { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(0); qjCluster = builder.Build(); MiniDFSCluster cluster = qjCluster.GetDfsCluster(); // No upgrade is in progress at the moment. CheckJnPreviousDirExistence(qjCluster, false); CheckClusterPreviousDirExistence(cluster, false); AssertCTimesEqual(cluster); // Transition NN0 to active and do some FS ops. cluster.TransitionToActive(0); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1"))); long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.ShutdownNameNode(1); cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade ); cluster.RestartNameNode(0, false); NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade <= GetCommittedTxnIdValue(qjCluster )); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2"))); CheckNnPreviousDirExistence(cluster, 0, true); CheckNnPreviousDirExistence(cluster, 1, false); CheckJnPreviousDirExistence(qjCluster, true); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration (1)); NUnit.Framework.Assert.AreEqual(0, rc); cluster.RestartNameNode(1); long cidDuringUpgrade = GetCommittedTxnIdValue(qjCluster); NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidBeforeUpgrade); RunFinalizeCommand(cluster); NUnit.Framework.Assert.AreEqual(cidDuringUpgrade, GetCommittedTxnIdValue(qjCluster )); CheckClusterPreviousDirExistence(cluster, false); CheckJnPreviousDirExistence(qjCluster, false); AssertCTimesEqual(cluster); } finally { if (fs != null) { fs.Close(); } if (qjCluster != null) { qjCluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/> public virtual void TestTwoActiveNNs() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).Build(); try { cluster.GetDfsCluster().WaitActive(); cluster.GetDfsCluster().TransitionToActive(0); DFSClient client0 = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress (), conf); DFSClient client1 = new DFSClient(cluster.GetDfsCluster().GetNameNode(1).GetNameNodeAddress (), conf); DFSInotifyEventInputStream eis = client0.GetInotifyEventStream(); for (int i = 0; i < 10; i++) { client0.Mkdirs("/dir" + i, null, false); } cluster.GetDfsCluster().TransitionToActive(1); for (int i_1 = 10; i_1 < 20; i_1++) { client1.Mkdirs("/dir" + i_1, null, false); } // make sure that the old active can't read any further than the edits // it logged itself (it has no idea whether the in-progress edits from // the other writer have actually been committed) EventBatch batch = null; for (int i_2 = 0; i_2 < 10; i_2++) { batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); NUnit.Framework.Assert.IsTrue(((Event.CreateEvent)batch.GetEvents()[0]).GetPath() .Equals("/dir" + i_2)); } NUnit.Framework.Assert.IsTrue(eis.Poll() == null); } finally { try { cluster.Shutdown(); } catch (ExitUtil.ExitException) { } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestCheckpoint() { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, 1); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); QueryForPreparation(dfs); dfs.Mkdirs(foo); long txid = dfs.RollEdits(); NUnit.Framework.Assert.IsTrue(txid > 0); int retries = 0; while (++retries < 5) { NNStorage storage = dfsCluster.GetNamesystem(1).GetFSImage().GetStorage(); if (storage.GetFsImageName(txid - 1) != null) { return; } Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.Fail("new checkpoint does not exist"); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestFinalize() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); FSImage fsimage = dfsCluster.GetNamesystem(0).GetFSImage(); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); dfs.Mkdirs(bar); QueryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. NUnit.Framework.Assert.IsTrue(fsimage.HasRollbackFSImage()); info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Finalize); NUnit.Framework.Assert.IsTrue(info.IsFinalized()); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); // Once finalized, there should be no more fsimage for rollbacks. NUnit.Framework.Assert.IsFalse(fsimage.HasRollbackFSImage()); // Should have no problem in restart and replaying edits that include // the FINALIZE op. dfsCluster.RestartNameNode(0); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestDowngrade() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); dfs.Mkdirs(bar); TestRollingUpgrade.QueryForPreparation(dfs); dfs.Close(); dfsCluster.RestartNameNode(0, true, "-rollingUpgrade", "downgrade"); // Once downgraded, there should be no more fsimage for rollbacks. NUnit.Framework.Assert.IsFalse(dfsCluster.GetNamesystem(0).GetFSImage().HasRollbackFSImage ()); // shutdown NN1 dfsCluster.ShutdownNameNode(1); dfsCluster.TransitionToActive(0); dfs = dfsCluster.GetFileSystem(0); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsTrue(dfs.Exists(bar)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void Setup() { Configuration conf = new Configuration(); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).Build(); cluster = miniQjmHaCluster.GetDfsCluster(); jCluster = miniQjmHaCluster.GetJournalCluster(); // make nn0 active cluster.TransitionToActive(0); // do sth to generate in-progress edit log data DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs (cluster, conf); dfs.Mkdirs(new Path("/test2")); dfs.Close(); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/> public virtual void TestNNFailover() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).Build(); try { cluster.GetDfsCluster().WaitActive(); cluster.GetDfsCluster().TransitionToActive(0); DFSClient client = ((DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster .GetDfsCluster(), conf)).dfs; DFSInotifyEventInputStream eis = client.GetInotifyEventStream(); for (int i = 0; i < 10; i++) { client.Mkdirs("/dir" + i, null, false); } cluster.GetDfsCluster().ShutdownNameNode(0); cluster.GetDfsCluster().TransitionToActive(1); EventBatch batch = null; // we can read all of the edits logged by the old active from the new // active for (int i_1 = 0; i_1 < 10; i_1++) { batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); NUnit.Framework.Assert.IsTrue(((Event.CreateEvent)batch.GetEvents()[0]).GetPath() .Equals("/dir" + i_1)); } NUnit.Framework.Assert.IsTrue(eis.Poll() == null); } finally { cluster.Shutdown(); } }
/// <summary>Tests all FsEditLogOps that are converted to inotify events.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/> public virtual void TestBasic() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); // so that we can get an atime change conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 1); MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(2); MiniQJMHACluster cluster = builder.Build(); try { cluster.GetDfsCluster().WaitActive(); cluster.GetDfsCluster().TransitionToActive(0); DFSClient client = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress (), conf); FileSystem fs = cluster.GetDfsCluster().GetFileSystem(0); DFSTestUtil.CreateFile(fs, new Path("/file"), BlockSize, (short)1, 0L); DFSTestUtil.CreateFile(fs, new Path("/file3"), BlockSize, (short)1, 0L); DFSTestUtil.CreateFile(fs, new Path("/file5"), BlockSize, (short)1, 0L); DFSInotifyEventInputStream eis = client.GetInotifyEventStream(); client.Rename("/file", "/file4", null); // RenameOp -> RenameEvent client.Rename("/file4", "/file2"); // RenameOldOp -> RenameEvent // DeleteOp, AddOp -> UnlinkEvent, CreateEvent OutputStream os = client.Create("/file2", true, (short)2, BlockSize); os.Write(new byte[BlockSize]); os.Close(); // CloseOp -> CloseEvent // AddOp -> AppendEvent os = client.Append("/file2", BlockSize, EnumSet.Of(CreateFlag.Append), null, null ); os.Write(new byte[BlockSize]); os.Close(); // CloseOp -> CloseEvent Sharpen.Thread.Sleep(10); // so that the atime will get updated on the next line client.Open("/file2").Read(new byte[1]); // TimesOp -> MetadataUpdateEvent // SetReplicationOp -> MetadataUpdateEvent client.SetReplication("/file2", (short)1); // ConcatDeleteOp -> AppendEvent, UnlinkEvent, CloseEvent client.Concat("/file2", new string[] { "/file3" }); client.Delete("/file2", false); // DeleteOp -> UnlinkEvent client.Mkdirs("/dir", null, false); // MkdirOp -> CreateEvent // SetPermissionsOp -> MetadataUpdateEvent client.SetPermission("/dir", FsPermission.ValueOf("-rw-rw-rw-")); // SetOwnerOp -> MetadataUpdateEvent client.SetOwner("/dir", "username", "groupname"); client.CreateSymlink("/dir", "/dir2", false); // SymlinkOp -> CreateEvent client.SetXAttr("/file5", "user.field", Sharpen.Runtime.GetBytesForString("value" ), EnumSet.Of(XAttrSetFlag.Create)); // SetXAttrOp -> MetadataUpdateEvent // RemoveXAttrOp -> MetadataUpdateEvent client.RemoveXAttr("/file5", "user.field"); // SetAclOp -> MetadataUpdateEvent client.SetAcl("/file5", AclEntry.ParseAclSpec("user::rwx,user:foo:rw-,group::r--,other::---" , true)); client.RemoveAcl("/file5"); // SetAclOp -> MetadataUpdateEvent client.Rename("/file5", "/dir"); // RenameOldOp -> RenameEvent EventBatch batch = null; // RenameOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); long txid = batch.GetTxid(); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.AreEqual("/file4", re.GetDstPath()); NUnit.Framework.Assert.AreEqual("/file", re.GetSrcPath()); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); long eventsBehind = eis.GetTxidsBehindEstimate(); // RenameOldOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re2 = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(re2.GetDstPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(re2.GetSrcPath().Equals("/file4")); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); // AddOp with overwrite batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce.GetiNodeType() == Event.CreateEvent.INodeType.File ); NUnit.Framework.Assert.IsTrue(ce.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce.GetReplication() > 0); NUnit.Framework.Assert.IsTrue(ce.GetSymlinkTarget() == null); NUnit.Framework.Assert.IsTrue(ce.GetOverwrite()); NUnit.Framework.Assert.AreEqual(BlockSize, ce.GetDefaultBlockSize()); // CloseOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Close); Event.CloseEvent ce2 = (Event.CloseEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce2.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce2.GetFileSize() > 0); NUnit.Framework.Assert.IsTrue(ce2.GetTimestamp() > 0); // AppendOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Append); Event.AppendEvent append2 = (Event.AppendEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.AreEqual("/file2", append2.GetPath()); NUnit.Framework.Assert.IsFalse(append2.ToNewBlock()); // CloseOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Close); NUnit.Framework.Assert.IsTrue(((Event.CloseEvent)batch.GetEvents()[0]).GetPath(). Equals("/file2")); // TimesOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(mue.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Times); // SetReplicationOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue2.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(mue2.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Replication); NUnit.Framework.Assert.IsTrue(mue2.GetReplication() == 1); // ConcatDeleteOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(3, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Append); NUnit.Framework.Assert.IsTrue(((Event.AppendEvent)batch.GetEvents()[0]).GetPath() .Equals("/file2")); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[1].GetEventType() == Event.EventType .Unlink); Event.UnlinkEvent ue2 = (Event.UnlinkEvent)batch.GetEvents()[1]; NUnit.Framework.Assert.IsTrue(ue2.GetPath().Equals("/file3")); NUnit.Framework.Assert.IsTrue(ue2.GetTimestamp() > 0); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[2].GetEventType() == Event.EventType .Close); Event.CloseEvent ce3 = (Event.CloseEvent)batch.GetEvents()[2]; NUnit.Framework.Assert.IsTrue(ce3.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce3.GetTimestamp() > 0); // DeleteOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Unlink); Event.UnlinkEvent ue = (Event.UnlinkEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ue.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ue.GetTimestamp() > 0); // MkdirOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce4 = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce4.GetiNodeType() == Event.CreateEvent.INodeType.Directory ); NUnit.Framework.Assert.IsTrue(ce4.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(ce4.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce4.GetReplication() == 0); NUnit.Framework.Assert.IsTrue(ce4.GetSymlinkTarget() == null); // SetPermissionsOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue3.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(mue3.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Perms); NUnit.Framework.Assert.IsTrue(mue3.GetPerms().ToString().Contains("rw-rw-rw-")); // SetOwnerOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue4.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(mue4.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Owner); NUnit.Framework.Assert.IsTrue(mue4.GetOwnerName().Equals("username")); NUnit.Framework.Assert.IsTrue(mue4.GetGroupName().Equals("groupname")); // SymlinkOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce5 = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce5.GetiNodeType() == Event.CreateEvent.INodeType.Symlink ); NUnit.Framework.Assert.IsTrue(ce5.GetPath().Equals("/dir2")); NUnit.Framework.Assert.IsTrue(ce5.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce5.GetReplication() == 0); NUnit.Framework.Assert.IsTrue(ce5.GetSymlinkTarget().Equals("/dir")); // SetXAttrOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue5.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue5.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Xattrs); NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs().Count == 1); NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs()[0].GetName().Contains("field")); NUnit.Framework.Assert.IsTrue(!mue5.IsxAttrsRemoved()); // RemoveXAttrOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue6.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue6.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Xattrs); NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs().Count == 1); NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs()[0].GetName().Contains("field")); NUnit.Framework.Assert.IsTrue(mue6.IsxAttrsRemoved()); // SetAclOp (1) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue7.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue7.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Acls); NUnit.Framework.Assert.IsTrue(mue7.GetAcls().Contains(AclEntry.ParseAclEntry("user::rwx" , true))); // SetAclOp (2) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue8.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue8.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Acls); NUnit.Framework.Assert.IsTrue(mue8.GetAcls() == null); // RenameOp (2) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re3 = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(re3.GetDstPath().Equals("/dir/file5")); NUnit.Framework.Assert.IsTrue(re3.GetSrcPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); // Returns null when there are no further events NUnit.Framework.Assert.IsTrue(eis.Poll() == null); // make sure the estimate hasn't changed since the above assertion // tells us that we are fully caught up to the current namesystem state // and we should not have been behind at all when eventsBehind was set // either, since there were few enough events that they should have all // been read to the client during the first poll() call NUnit.Framework.Assert.IsTrue(eis.GetTxidsBehindEstimate() == eventsBehind); } finally { cluster.Shutdown(); } }
public virtual void TestRollbackWithHAQJM() { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).Build(); MiniDFSCluster dfsCluster = cluster.GetDfsCluster(); dfsCluster.WaitActive(); // let NN1 tail editlog every 1s dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); dfsCluster.RestartNameNode(1); dfsCluster.TransitionToActive(0); DistributedFileSystem dfs = dfsCluster.GetFileSystem(0); dfs.Mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare ); NUnit.Framework.Assert.IsTrue(info.IsStarted()); // create new directory dfs.Mkdirs(bar); dfs.Close(); TestRollingUpgrade.QueryForPreparation(dfs); // If the query returns true, both active and the standby NN should have // rollback fsimage ready. NUnit.Framework.Assert.IsTrue(dfsCluster.GetNameNode(0).GetFSImage().HasRollbackFSImage ()); NUnit.Framework.Assert.IsTrue(dfsCluster.GetNameNode(1).GetFSImage().HasRollbackFSImage ()); // rollback NN0 dfsCluster.RestartNameNode(0, true, "-rollingUpgrade", "rollback"); // shutdown NN1 dfsCluster.ShutdownNameNode(1); dfsCluster.TransitionToActive(0); // make sure /foo is still there, but /bar is not dfs = dfsCluster.GetFileSystem(0); NUnit.Framework.Assert.IsTrue(dfs.Exists(foo)); NUnit.Framework.Assert.IsFalse(dfs.Exists(bar)); // check the details of NNStorage NNStorage storage = dfsCluster.GetNamesystem(0).GetFSImage().GetStorage(); // segments:(startSegment, mkdir, start upgrade endSegment), // (startSegment, mkdir, endSegment) CheckNNStorage(storage, 4, 7); // check storage in JNs for (int i = 0; i < NumJournalNodes; i++) { FilePath dir = cluster.GetJournalCluster().GetCurrentDir(0, MiniQJMHACluster.Nameservice ); CheckJNStorage(dir, 5, 7); } // restart NN0 again to make sure we can start using the new fsimage and // the corresponding md5 checksum dfsCluster.RestartNameNode(0); // start the rolling upgrade again to make sure we do not load upgrade // status after the rollback dfsCluster.TransitionToActive(0); dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestRollbackWithJournalNodes() { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(0); qjCluster = builder.Build(); MiniDFSCluster cluster = qjCluster.GetDfsCluster(); // No upgrade is in progress at the moment. CheckClusterPreviousDirExistence(cluster, false); AssertCTimesEqual(cluster); CheckJnPreviousDirExistence(qjCluster, false); // Transition NN0 to active and do some FS ops. cluster.TransitionToActive(0); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1"))); long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.ShutdownNameNode(1); cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade ); cluster.RestartNameNode(0, false); CheckNnPreviousDirExistence(cluster, 0, true); CheckNnPreviousDirExistence(cluster, 1, false); CheckJnPreviousDirExistence(qjCluster, true); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2"))); long cidDuringUpgrade = GetCommittedTxnIdValue(qjCluster); NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidBeforeUpgrade); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration (1)); NUnit.Framework.Assert.AreEqual(0, rc); cluster.RestartNameNode(1); CheckNnPreviousDirExistence(cluster, 0, true); CheckNnPreviousDirExistence(cluster, 1, true); CheckJnPreviousDirExistence(qjCluster, true); AssertCTimesEqual(cluster); // Shut down the NNs, but deliberately leave the JNs up and running. ICollection <URI> nn1NameDirs = cluster.GetNameDirs(0); cluster.Shutdown(); conf.SetStrings(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(nn1NameDirs )); NameNode.DoRollback(conf, false); long cidAfterRollback = GetCommittedTxnIdValue(qjCluster); NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade < cidAfterRollback); // make sure the committedTxnId has been reset correctly after rollback NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidAfterRollback); // The rollback operation should have rolled back the first NN's local // dirs, and the shared dir, but not the other NN's dirs. Those have to be // done by bootstrapping the standby. CheckNnPreviousDirExistence(cluster, 0, false); CheckJnPreviousDirExistence(qjCluster, false); } finally { if (fs != null) { fs.Close(); } if (qjCluster != null) { qjCluster.Shutdown(); } } }
public virtual void TestUpgradeWithJournalNodes() { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(0); qjCluster = builder.Build(); MiniDFSCluster cluster = qjCluster.GetDfsCluster(); // No upgrade is in progress at the moment. CheckJnPreviousDirExistence(qjCluster, false); CheckClusterPreviousDirExistence(cluster, false); AssertCTimesEqual(cluster); // Transition NN0 to active and do some FS ops. cluster.TransitionToActive(0); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1"))); // get the value of the committedTxnId in journal nodes long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.ShutdownNameNode(1); cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade ); cluster.RestartNameNode(0, false); CheckNnPreviousDirExistence(cluster, 0, true); CheckNnPreviousDirExistence(cluster, 1, false); CheckJnPreviousDirExistence(qjCluster, true); NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade <= GetCommittedTxnIdValue(qjCluster )); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2"))); // Restart NN0 without the -upgrade flag, to make sure that works. cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Regular ); cluster.RestartNameNode(0, false); // Make sure we can still do FS ops after upgrading. cluster.TransitionToActive(0); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo3"))); NUnit.Framework.Assert.IsTrue(GetCommittedTxnIdValue(qjCluster) > cidBeforeUpgrade ); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration (1)); NUnit.Framework.Assert.AreEqual(0, rc); // Now restart NN1 and make sure that we can do ops against that as well. cluster.RestartNameNode(1); cluster.TransitionToStandby(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo4"))); AssertCTimesEqual(cluster); } finally { if (fs != null) { fs.Close(); } if (qjCluster != null) { qjCluster.Shutdown(); } } }