/// <exception cref="System.Exception"/>
public virtual int Run(string[] argv)
{
string description = "hdfs lsSnapshottableDir: \n" + "\tGet the list of snapshottable directories that are owned by the current user.\n"
+ "\tReturn all the snapshottable directories if the current user is a super user.\n";
if (argv.Length != 0)
{
System.Console.Error.WriteLine("Usage: \n" + description);
return(1);
}
FileSystem fs = FileSystem.Get(GetConf());
if (!(fs is DistributedFileSystem))
{
System.Console.Error.WriteLine("LsSnapshottableDir can only be used in DistributedFileSystem"
);
return(1);
}
DistributedFileSystem dfs = (DistributedFileSystem)fs;
try
{
SnapshottableDirectoryStatus[] stats = dfs.GetSnapshottableDirListing();
SnapshottableDirectoryStatus.Print(stats, System.Console.Out);
}
catch (IOException e)
{
string[] content = e.GetLocalizedMessage().Split("\n");
System.Console.Error.WriteLine("lsSnapshottableDir: " + content[0]);
return(1);
}
return(0);
}
public virtual void TestSnapshottableDirs()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 0, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 0L, MetricsAsserts.GetMetrics(NnMetrics
));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 0L, MetricsAsserts.GetMetrics
(NnMetrics));
// Allow snapshots for directories, and check the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
Path sub2 = new Path(dir, "sub2");
Path file = new Path(sub2, "file");
DFSTestUtil.CreateFile(hdfs, file, 1024, Replication, seed);
hdfs.AllowSnapshot(sub2);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 2L, MetricsAsserts.GetMetrics(NnMetrics
));
Path subsub1 = new Path(sub1, "sub1sub1");
Path subfile = new Path(subsub1, "file");
DFSTestUtil.CreateFile(hdfs, subfile, 1024, Replication, seed);
hdfs.AllowSnapshot(subsub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 3L, MetricsAsserts.GetMetrics(NnMetrics
));
// Set an already snapshottable directory to snapshottable, should not
// change the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
// But the number of allowSnapshot operations still increases
MetricsAsserts.AssertCounter("AllowSnapshotOps", 4L, MetricsAsserts.GetMetrics(NnMetrics
));
// Disallow the snapshot for snapshottable directories, then check the
// metrics again
hdfs.DisallowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
// delete subsub1, snapshottable directories should be 1
hdfs.Delete(subsub1, true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
// list all the snapshottable directories
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
MetricsAsserts.AssertCounter("ListSnapshottableDirOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
}
/// <summary>
/// Test the listing with different user names to make sure only directories
/// that are owned by the user are listed.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestListWithDifferentUser()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// first make dir1 and dir2 snapshottable
hdfs.AllowSnapshot(dir1);
hdfs.AllowSnapshot(dir2);
hdfs.SetPermission(root, FsPermission.ValueOf("-rwxrwxrwx"));
// create two dirs and make them snapshottable under the name of user1
UserGroupInformation ugi1 = UserGroupInformation.CreateUserForTesting("user1", new
string[] { "group1" });
DistributedFileSystem fs1 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi1
, conf);
Path dir1_user1 = new Path("/dir1_user1");
Path dir2_user1 = new Path("/dir2_user1");
fs1.Mkdirs(dir1_user1);
fs1.Mkdirs(dir2_user1);
hdfs.AllowSnapshot(dir1_user1);
hdfs.AllowSnapshot(dir2_user1);
// user2
UserGroupInformation ugi2 = UserGroupInformation.CreateUserForTesting("user2", new
string[] { "group2" });
DistributedFileSystem fs2 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(ugi2
, conf);
Path dir_user2 = new Path("/dir_user2");
Path subdir_user2 = new Path(dir_user2, "subdir");
fs2.Mkdirs(dir_user2);
fs2.Mkdirs(subdir_user2);
hdfs.AllowSnapshot(dir_user2);
hdfs.AllowSnapshot(subdir_user2);
// super user
string supergroup = conf.Get(DFSConfigKeys.DfsPermissionsSuperusergroupKey, DFSConfigKeys
.DfsPermissionsSuperusergroupDefault);
UserGroupInformation superUgi = UserGroupInformation.CreateUserForTesting("superuser"
, new string[] { supergroup });
DistributedFileSystem fs3 = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(superUgi
, conf);
// list the snapshottable dirs for superuser
SnapshottableDirectoryStatus[] dirs = fs3.GetSnapshottableDirListing();
// 6 snapshottable dirs: dir1, dir2, dir1_user1, dir2_user1, dir_user2, and
// subdir_user2
NUnit.Framework.Assert.AreEqual(6, dirs.Length);
// list the snapshottable dirs for user1
dirs = fs1.GetSnapshottableDirListing();
// 2 dirs owned by user1: dir1_user1 and dir2_user1
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1_user1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2_user1, dirs[1].GetFullPath());
// list the snapshottable dirs for user2
dirs = fs2.GetSnapshottableDirListing();
// 2 dirs owned by user2: dir_user2 and subdir_user2
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir_user2, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(subdir_user2, dirs[1].GetFullPath());
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
/// <exception cref="System.IO.IOException"/>
internal virtual void CheckImage(int s)
{
string name = "s" + s;
// dump the fsdir tree
FilePath fsnBefore = DumpTree2File(name + "_before");
// save the namesystem to a temp file
FilePath imageFile = SaveFSImageToTempFile();
long numSdirBefore = fsn.GetNumSnapshottableDirs();
long numSnapshotBefore = fsn.GetNumSnapshots();
SnapshottableDirectoryStatus[] dirBefore = hdfs.GetSnapshottableDirListing();
// shutdown the cluster
cluster.Shutdown();
// dump the fsdir tree
FilePath fsnBetween = DumpTree2File(name + "_between");
SnapshotTestHelper.CompareDumpedTreeInFile(fsnBefore, fsnBetween, true);
// restart the cluster, and format the cluster
cluster = new MiniDFSCluster.Builder(conf).Format(true).NumDataNodes(Replication)
.Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
// load the namesystem from the temp file
LoadFSImageFromTempFile(imageFile);
// dump the fsdir tree again
FilePath fsnAfter = DumpTree2File(name + "_after");
// compare two dumped tree
SnapshotTestHelper.CompareDumpedTreeInFile(fsnBefore, fsnAfter, true);
long numSdirAfter = fsn.GetNumSnapshottableDirs();
long numSnapshotAfter = fsn.GetNumSnapshots();
SnapshottableDirectoryStatus[] dirAfter = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(numSdirBefore, numSdirAfter);
NUnit.Framework.Assert.AreEqual(numSnapshotBefore, numSnapshotAfter);
NUnit.Framework.Assert.AreEqual(dirBefore.Length, dirAfter.Length);
IList <string> pathListBefore = new AList <string>();
foreach (SnapshottableDirectoryStatus sBefore in dirBefore)
{
pathListBefore.AddItem(sBefore.GetFullPath().ToString());
}
foreach (SnapshottableDirectoryStatus sAfter in dirAfter)
{
NUnit.Framework.Assert.IsTrue(pathListBefore.Contains(sAfter.GetFullPath().ToString
()));
}
}
/// <summary>Test listing all the snapshottable directories</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// Initially there is no snapshottable directories in the system
SnapshottableDirectoryStatus[] dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make root as snapshottable
Path root = new Path("/");
hdfs.AllowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(string.Empty, dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(root, dirs[0].GetFullPath());
// Make root non-snaphsottable
hdfs.DisallowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make dir1 as snapshottable
hdfs.AllowSnapshot(dir1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// There is no snapshot for dir1 yet
NUnit.Framework.Assert.AreEqual(0, dirs[0].GetSnapshotNumber());
// Make dir2 as snapshottable
hdfs.AllowSnapshot(dir2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[1].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
// There is no snapshot for dir2 yet
NUnit.Framework.Assert.AreEqual(0, dirs[1].GetSnapshotNumber());
// Create dir3
Path dir3 = new Path("/TestSnapshot3");
hdfs.Mkdirs(dir3);
// Rename dir3 to dir2
hdfs.Rename(dir3, dir2, Options.Rename.Overwrite);
// Now we only have one snapshottable dir: dir1
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// Make dir2 snapshottable again
hdfs.AllowSnapshot(dir2);
// Create a snapshot for dir2
hdfs.CreateSnapshot(dir2, "s1");
hdfs.CreateSnapshot(dir2, "s2");
dirs = hdfs.GetSnapshottableDirListing();
// There are now 2 snapshots for dir2
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(2, dirs[1].GetSnapshotNumber());
// Create sub-dirs under dir1
Path sub1 = new Path(dir1, "sub1");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(dir1, "sub2");
Path file2 = new Path(sub2, "file2");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, seed);
// Make sub1 and sub2 snapshottable
hdfs.AllowSnapshot(sub1);
hdfs.AllowSnapshot(sub2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(4, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub1, dirs[2].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[3].GetFullPath());
// reset sub1
hdfs.DisallowSnapshot(sub1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(3, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[2].GetFullPath());
// Remove dir1, both dir1 and sub2 will be removed
hdfs.Delete(dir1, true);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[0].GetFullPath());
}