public virtual void TestOpenFilesWithRename()
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
// check for zero sized blocks
Path fileWithEmptyBlock = new Path("/test/test/test4");
fs.Create(fileWithEmptyBlock);
NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc();
string clientName = fs.GetClient().GetClientName();
// create one empty block
nameNodeRpc.AddBlock(fileWithEmptyBlock.ToString(), clientName, null, null, INodeId
.GrandfatherInodeId, null);
fs.CreateSnapshot(path, "s2");
fs.Rename(new Path("/test/test"), new Path("/test/test-renamed"));
fs.Delete(new Path("/test/test-renamed"), true);
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
cluster.RestartNameNode(true);
}
/// <summary>Restart the cluster, optionally saving a new checkpoint.</summary>
/// <param name="checkpoint">boolean true to save a new checkpoint</param>
/// <exception cref="System.Exception">if restart fails</exception>
private static void Restart(bool checkpoint)
{
NameNode nameNode = cluster.GetNameNode();
if (checkpoint)
{
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
}
Shutdown();
InitCluster(false);
}
public virtual void TestBlocksDeletedInEditLog()
{
Banner("Starting with NN0 active and NN1 standby, creating some blocks");
// Make 4 blocks persisted in the image.
DFSTestUtil.CreateFile(fs, new Path("/test"), 4 * BlockSize, (short)3, 1L);
NameNodeAdapter.EnterSafeMode(nn0, false);
NameNodeAdapter.SaveNamespace(nn0);
NameNodeAdapter.LeaveSafeMode(nn0);
// OP_ADD for 2 blocks
DFSTestUtil.CreateFile(fs, new Path("/test2"), 2 * BlockSize, (short)3, 1L);
// OP_DELETE for 4 blocks
fs.Delete(new Path("/test"), true);
RestartActive();
}
/// <exception cref="System.IO.IOException"/>
private void DoTestMultipleSnapshots(bool saveNamespace)
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
fs.CreateSnapshot(path, "s2");
fs.Delete(new Path("/test/test"), true);
fs.DeleteSnapshot(path, "s2");
cluster.TriggerBlockReports();
if (saveNamespace)
{
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
}
cluster.RestartNameNode(true);
}
public virtual void TestWithCheckpoint()
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
fs.Delete(new Path("/test/test"), true);
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
cluster.RestartNameNode(true);
// read snapshot file after restart
string test2snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(path.ToString(), "s1/test/test2");
DFSTestUtil.ReadFile(fs, new Path(test2snapshotPath));
string test3snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(path.ToString(), "s1/test/test3");
DFSTestUtil.ReadFile(fs, new Path(test3snapshotPath));
}
/*
* Try to read the files inside snapshot but renamed to different file and
* deleted after restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadRenamedSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
Path foo2 = new Path("/foo2");
hdfs.Mkdirs(foo);
hdfs.Mkdirs(foo2);
hdfs.AllowSnapshot(foo);
hdfs.AllowSnapshot(foo2);
Path bar = new Path(foo, "bar");
Path bar2 = new Path(foo2, "bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
// rename to another snapshottable directory and take snapshot
NUnit.Framework.Assert.IsTrue(hdfs.Rename(bar, bar2));
hdfs.CreateSnapshot(foo2, "s2");
// delete the original renamed file to make sure blocks are not updated by
// the original file
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar2, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
// file in first snapshot
string barSnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(barSnapshotPath));
// file in second snapshot after rename+delete
string bar2SnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(foo2.ToString(), "s2/bar");
DFSTestUtil.ReadFile(hdfs, new Path(bar2SnapshotPath));
}
public virtual void TestNoPopulatingReplQueuesWhenExitingSafemode()
{
DFSTestUtil.CreateFile(fs, new Path("/test"), 15 * BlockSize, (short)3, 1L);
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
// get some blocks in the SBN's image
nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false);
NameNodeAdapter.SaveNamespace(nn1);
nn1.GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false);
// and some blocks in the edit logs
DFSTestUtil.CreateFile(fs, new Path("/test2"), 15 * BlockSize, (short)3, 1L);
nn0.GetRpcServer().RollEditLog();
cluster.StopDataNode(1);
cluster.ShutdownNameNode(1);
//Configuration sbConf = cluster.getConfiguration(1);
//sbConf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 1);
cluster.RestartNameNode(1, false);
nn1 = cluster.GetNameNode(1);
GenericTestUtils.WaitFor(new _Supplier_708(this), 100, 10000);
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetUnderReplicatedBlocks(
));
NUnit.Framework.Assert.AreEqual(0L, nn1.GetNamesystem().GetPendingReplicationBlocks
());
}
public virtual void TestDownloadingLaterCheckpoint()
{
// Roll edit logs a few times to inflate txid
nn0.GetRpcServer().RollEditLog();
nn0.GetRpcServer().RollEditLog();
// Make checkpoint
NameNodeAdapter.EnterSafeMode(nn0, false);
NameNodeAdapter.SaveNamespace(nn0);
NameNodeAdapter.LeaveSafeMode(nn0);
long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId
();
NUnit.Framework.Assert.AreEqual(6, expectedCheckpointTxId);
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
// Should have copied over the namespace from the active
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
FSImageTestUtil.AssertNNFilesMatch(cluster);
// We should now be able to start the standby successfully.
cluster.RestartNameNode(1);
}
/*
* Try to read the files inside snapshot but deleted in original place after
* restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
hdfs.Mkdirs(foo);
hdfs.AllowSnapshot(foo);
Path bar = new Path("/foo/bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
string snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotPath
(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(snapshotPath));
}
public virtual void TestCheckpoint()
{
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
secondary = new SecondaryNameNode(conf);
SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager();
SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager
();
FileSystem fs = cluster.GetFileSystem();
HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 1. Create a snapshottable directory foo on the NN.
fs.Mkdirs(TestPath);
admin.AllowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 2. Create a snapshot of the dir foo. This will be referenced both in
// the SnapshotManager as well as in the file system tree. The snapshot
// count will go up to 1.
Path snapshotPath = fs.CreateSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 3. Start up a 2NN and have it do a checkpoint. It will have foo and its
// snapshot in its list of snapshottable dirs referenced from the
// SnapshotManager, as well as in the file system tree.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 4. Disallow snapshots on and delete foo on the NN. The snapshot count
// will go down to 0 and the snapshottable dir will be removed from the fs
// tree.
fs.DeleteSnapshot(TestPath, snapshotPath.GetName());
admin.DisallowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
// 5. Have the NN do a saveNamespace, writing out a new fsimage with
// snapshot count 0.
NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false);
NameNodeAdapter.SaveNamespace(cluster.GetNameNode());
NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode());
// 6. Have the still-running 2NN do a checkpoint. It will notice that the
// fsimage has changed on the NN and redownload/reload from that image.
// This will replace all INodes in the file system tree as well as reset
// the snapshot counter to 0 in the SnapshotManager. However, it will not
// clear the list of snapshottable dirs referenced from the
// SnapshotManager. When it writes out an fsimage, the 2NN will write out
// 0 for the snapshot count, but still serialize the snapshottable dir
// referenced in the SnapshotManager even though it no longer appears in
// the file system tree. The NN will not be able to start up with this.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (secondary != null)
{
secondary.Shutdown();
}
}
}
