public virtual void TestOpenFilesWithRename() { Path path = new Path("/test"); DoWriteAndAbort(fs, path); // check for zero sized blocks Path fileWithEmptyBlock = new Path("/test/test/test4"); fs.Create(fileWithEmptyBlock); NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc(); string clientName = fs.GetClient().GetClientName(); // create one empty block nameNodeRpc.AddBlock(fileWithEmptyBlock.ToString(), clientName, null, null, INodeId .GrandfatherInodeId, null); fs.CreateSnapshot(path, "s2"); fs.Rename(new Path("/test/test"), new Path("/test/test-renamed")); fs.Delete(new Path("/test/test-renamed"), true); NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); cluster.RestartNameNode(true); }
/// <summary>Restart the cluster, optionally saving a new checkpoint.</summary> /// <param name="checkpoint">boolean true to save a new checkpoint</param> /// <exception cref="System.Exception">if restart fails</exception> private static void Restart(bool checkpoint) { NameNode nameNode = cluster.GetNameNode(); if (checkpoint) { NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); } Shutdown(); InitCluster(false); }
public virtual void TestBlocksDeletedInEditLog() { Banner("Starting with NN0 active and NN1 standby, creating some blocks"); // Make 4 blocks persisted in the image. DFSTestUtil.CreateFile(fs, new Path("/test"), 4 * BlockSize, (short)3, 1L); NameNodeAdapter.EnterSafeMode(nn0, false); NameNodeAdapter.SaveNamespace(nn0); NameNodeAdapter.LeaveSafeMode(nn0); // OP_ADD for 2 blocks DFSTestUtil.CreateFile(fs, new Path("/test2"), 2 * BlockSize, (short)3, 1L); // OP_DELETE for 4 blocks fs.Delete(new Path("/test"), true); RestartActive(); }
/// <exception cref="System.IO.IOException"/> private void DoTestMultipleSnapshots(bool saveNamespace) { Path path = new Path("/test"); DoWriteAndAbort(fs, path); fs.CreateSnapshot(path, "s2"); fs.Delete(new Path("/test/test"), true); fs.DeleteSnapshot(path, "s2"); cluster.TriggerBlockReports(); if (saveNamespace) { NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); } cluster.RestartNameNode(true); }
public virtual void TestWithCheckpoint() { Path path = new Path("/test"); DoWriteAndAbort(fs, path); fs.Delete(new Path("/test/test"), true); NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); cluster.RestartNameNode(true); // read snapshot file after restart string test2snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .GetSnapshotPath(path.ToString(), "s1/test/test2"); DFSTestUtil.ReadFile(fs, new Path(test2snapshotPath)); string test3snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .GetSnapshotPath(path.ToString(), "s1/test/test3"); DFSTestUtil.ReadFile(fs, new Path(test3snapshotPath)); }
/* * Try to read the files inside snapshot but renamed to different file and * deleted after restarting post checkpoint. refer HDFS-5427 */ /// <exception cref="System.Exception"/> public virtual void TestReadRenamedSnapshotFileWithCheckpoint() { Path foo = new Path("/foo"); Path foo2 = new Path("/foo2"); hdfs.Mkdirs(foo); hdfs.Mkdirs(foo2); hdfs.AllowSnapshot(foo); hdfs.AllowSnapshot(foo2); Path bar = new Path(foo, "bar"); Path bar2 = new Path(foo2, "bar"); DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L); hdfs.CreateSnapshot(foo, "s1"); // rename to another snapshottable directory and take snapshot NUnit.Framework.Assert.IsTrue(hdfs.Rename(bar, bar2)); hdfs.CreateSnapshot(foo2, "s2"); // delete the original renamed file to make sure blocks are not updated by // the original file NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar2, true)); // checkpoint NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); // restart namenode to load snapshot files from fsimage cluster.RestartNameNode(true); // file in first snapshot string barSnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .GetSnapshotPath(foo.ToString(), "s1/bar"); DFSTestUtil.ReadFile(hdfs, new Path(barSnapshotPath)); // file in second snapshot after rename+delete string bar2SnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .GetSnapshotPath(foo2.ToString(), "s2/bar"); DFSTestUtil.ReadFile(hdfs, new Path(bar2SnapshotPath)); }
public virtual void TestNoPopulatingReplQueuesWhenExitingSafemode() { DFSTestUtil.CreateFile(fs, new Path("/test"), 15 * BlockSize, (short)3, 1L); HATestUtil.WaitForStandbyToCatchUp(nn0, nn1); // get some blocks in the SBN's image nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); NameNodeAdapter.SaveNamespace(nn1); nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); // and some blocks in the edit logs DFSTestUtil.CreateFile(fs, new Path("/test2"), 15 * BlockSize, (short)3, 1L); nn0.GetRpcServer().RollEditLog(); cluster.StopDataNode(1); cluster.ShutdownNameNode(1); //Configuration sbConf = cluster.getConfiguration(1); //sbConf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 1); cluster.RestartNameNode(1, false); nn1 = cluster.GetNameNode(1); GenericTestUtils.WaitFor(new _Supplier_708(this), 100, 10000); BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager()); NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetUnderReplicatedBlocks( )); NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetPendingReplicationBlocks ()); }
public virtual void TestDownloadingLaterCheckpoint() { // Roll edit logs a few times to inflate txid nn0.GetRpcServer().RollEditLog(); nn0.GetRpcServer().RollEditLog(); // Make checkpoint NameNodeAdapter.EnterSafeMode(nn0, false); NameNodeAdapter.SaveNamespace(nn0); NameNodeAdapter.LeaveSafeMode(nn0); long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId (); NUnit.Framework.Assert.AreEqual(6, expectedCheckpointTxId); int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration (1)); NUnit.Framework.Assert.AreEqual(0, rc); // Should have copied over the namespace from the active FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId )); FSImageTestUtil.AssertNNFilesMatch(cluster); // We should now be able to start the standby successfully. cluster.RestartNameNode(1); }
/* * Try to read the files inside snapshot but deleted in original place after * restarting post checkpoint. refer HDFS-5427 */ /// <exception cref="System.Exception"/> public virtual void TestReadSnapshotFileWithCheckpoint() { Path foo = new Path("/foo"); hdfs.Mkdirs(foo); hdfs.AllowSnapshot(foo); Path bar = new Path("/foo/bar"); DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L); hdfs.CreateSnapshot(foo, "s1"); NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar, true)); // checkpoint NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); // restart namenode to load snapshot files from fsimage cluster.RestartNameNode(true); string snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotPath (foo.ToString(), "s1/bar"); DFSTestUtil.ReadFile(hdfs, new Path(snapshotPath)); }
public virtual void TestCheckpoint() { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); secondary = new SecondaryNameNode(conf); SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager(); SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager (); FileSystem fs = cluster.GetFileSystem(); HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 1. Create a snapshottable directory foo on the NN. fs.Mkdirs(TestPath); admin.AllowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 2. Create a snapshot of the dir foo. This will be referenced both in // the SnapshotManager as well as in the file system tree. The snapshot // count will go up to 1. Path snapshotPath = fs.CreateSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 3. Start up a 2NN and have it do a checkpoint. It will have foo and its // snapshot in its list of snapshottable dirs referenced from the // SnapshotManager, as well as in the file system tree. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 4. Disallow snapshots on and delete foo on the NN. The snapshot count // will go down to 0 and the snapshottable dir will be removed from the fs // tree. fs.DeleteSnapshot(TestPath, snapshotPath.GetName()); admin.DisallowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); // 5. Have the NN do a saveNamespace, writing out a new fsimage with // snapshot count 0. NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false); NameNodeAdapter.SaveNamespace(cluster.GetNameNode()); NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode()); // 6. Have the still-running 2NN do a checkpoint. It will notice that the // fsimage has changed on the NN and redownload/reload from that image. // This will replace all INodes in the file system tree as well as reset // the snapshot counter to 0 in the SnapshotManager. However, it will not // clear the list of snapshottable dirs referenced from the // SnapshotManager. When it writes out an fsimage, the 2NN will write out // 0 for the snapshot count, but still serialize the snapshottable dir // referenced in the SnapshotManager even though it no longer appears in // the file system tree. The NN will not be able to start up with this. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); } finally { if (cluster != null) { cluster.Shutdown(); } if (secondary != null) { secondary.Shutdown(); } } }