/// <summary>Test snapshot after file appending</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotAfterAppending()
{
Path file = new Path(dir, "file");
// 1. create snapshot --> create file --> append
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
// 2. create snapshot --> modify the file --> append
hdfs.CreateSnapshot(dir, "s1");
hdfs.SetReplication(file, (short)(Replication - 1));
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize());
// 3. create snapshot --> append
hdfs.CreateSnapshot(dir, "s2");
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize());
}
/// <summary>
/// Create/modify/delete files under a given directory, also create snapshots
/// of directories.
/// </summary>
/// <exception cref="System.Exception"/>
private void ModifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
{
Path file10 = new Path(modifyDir, "file10");
Path file11 = new Path(modifyDir, "file11");
Path file12 = new Path(modifyDir, "file12");
Path file13 = new Path(modifyDir, "file13");
Path link13 = new Path(modifyDir, "link13");
Path file14 = new Path(modifyDir, "file14");
Path file15 = new Path(modifyDir, "file15");
DFSTestUtil.CreateFile(hdfs, file10, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file11, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file12, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file13, Blocksize, Replication1, seed);
// create link13
hdfs.CreateSymlink(file13, link13, false);
// create snapshot
foreach (Path snapshotDir in snapshotDirs)
{
hdfs.AllowSnapshot(snapshotDir);
hdfs.CreateSnapshot(snapshotDir, GenSnapshotName(snapshotDir));
}
// delete file11
hdfs.Delete(file11, true);
// modify file12
hdfs.SetReplication(file12, Replication);
// modify file13
hdfs.SetReplication(file13, Replication);
// delete link13
hdfs.Delete(link13, false);
// create file14
DFSTestUtil.CreateFile(hdfs, file14, Blocksize, Replication, seed);
// create file15
DFSTestUtil.CreateFile(hdfs, file15, Blocksize, Replication, seed);
// create snapshot
foreach (Path snapshotDir_1 in snapshotDirs)
{
hdfs.CreateSnapshot(snapshotDir_1, GenSnapshotName(snapshotDir_1));
}
// create file11 again
DFSTestUtil.CreateFile(hdfs, file11, Blocksize, Replication, seed);
// delete file12
hdfs.Delete(file12, true);
// modify file13
hdfs.SetReplication(file13, (short)(Replication - 2));
// create link13 again
hdfs.CreateSymlink(file13, link13, false);
// delete file14
hdfs.Delete(file14, true);
// modify file15
hdfs.SetReplication(file15, (short)(Replication - 1));
// create snapshot
foreach (Path snapshotDir_2 in snapshotDirs)
{
hdfs.CreateSnapshot(snapshotDir_2, GenSnapshotName(snapshotDir_2));
}
// modify file10
hdfs.SetReplication(file10, (short)(Replication + 1));
}
public virtual void TestSnapshots()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
MetricsAsserts.AssertGauge("Snapshots", 0, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("CreateSnapshotOps", 0L, MetricsAsserts.GetMetrics(NnMetrics
));
// Create a snapshot for a non-snapshottable directory, thus should not
// change the metrics
try
{
hdfs.CreateSnapshot(sub1, "s1");
}
catch (Exception)
{
}
MetricsAsserts.AssertGauge("Snapshots", 0, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("CreateSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
// Create snapshot for sub1
hdfs.AllowSnapshot(sub1);
hdfs.CreateSnapshot(sub1, "s1");
MetricsAsserts.AssertGauge("Snapshots", 1, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("CreateSnapshotOps", 2L, MetricsAsserts.GetMetrics(NnMetrics
));
hdfs.CreateSnapshot(sub1, "s2");
MetricsAsserts.AssertGauge("Snapshots", 2, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("CreateSnapshotOps", 3L, MetricsAsserts.GetMetrics(NnMetrics
));
hdfs.GetSnapshotDiffReport(sub1, "s1", "s2");
MetricsAsserts.AssertCounter("SnapshotDiffReportOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
// Create snapshot for a directory under sub1
Path subsub1 = new Path(sub1, "sub1sub1");
Path subfile = new Path(subsub1, "file");
DFSTestUtil.CreateFile(hdfs, subfile, 1024, Replication, seed);
hdfs.AllowSnapshot(subsub1);
hdfs.CreateSnapshot(subsub1, "s11");
MetricsAsserts.AssertGauge("Snapshots", 3, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("CreateSnapshotOps", 4L, MetricsAsserts.GetMetrics(NnMetrics
));
// delete snapshot
hdfs.DeleteSnapshot(sub1, "s2");
MetricsAsserts.AssertGauge("Snapshots", 2, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("DeleteSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
// rename snapshot
hdfs.RenameSnapshot(sub1, "s1", "NewS1");
MetricsAsserts.AssertGauge("Snapshots", 2, MetricsAsserts.GetMetrics(NsMetrics));
MetricsAsserts.AssertCounter("RenameSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
}
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotWithInvalidName()
{
Path file1 = new Path(dir1, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(dir1);
try
{
hdfs.CreateSnapshot(dir1, snapshot1);
}
catch (RemoteException)
{
}
}
public virtual void TestSnapshotOnRoot()
{
Path root = new Path("/");
hdfs.AllowSnapshot(root);
hdfs.CreateSnapshot(root, "s1");
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
INodeDirectory rootNode = fsn.dir.GetINode4Write(root.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue("The children list of root should be empty", rootNode
.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId
).IsEmpty());
// one snapshot on root: s1
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = rootNode.GetDiffs().
AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s1 = rootNode.GetSnapshot
(DFSUtil.String2Bytes("s1"));
NUnit.Framework.Assert.AreEqual(s1.GetId(), diffList[0].GetSnapshotId());
// check SnapshotManager's snapshottable directory list
NUnit.Framework.Assert.AreEqual(1, fsn.GetSnapshotManager().GetNumSnapshottableDirs
());
SnapshottableDirectoryStatus[] sdirs = fsn.GetSnapshotManager().GetSnapshottableDirListing
(null);
NUnit.Framework.Assert.AreEqual(root, sdirs[0].GetFullPath());
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
}
/// <exception cref="System.IO.IOException"/>
private void DoTestMultipleSnapshots(bool saveNamespace)
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
fs.CreateSnapshot(path, "s2");
fs.Delete(new Path("/test/test"), true);
fs.DeleteSnapshot(path, "s2");
cluster.TriggerBlockReports();
if (saveNamespace)
{
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
}
cluster.RestartNameNode(true);
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
/// <summary>Create snapshot for a dir using a given snapshot name</summary>
/// <param name="hdfs">DistributedFileSystem instance</param>
/// <param name="snapshotRoot">The dir to be snapshotted</param>
/// <param name="snapshotName">The name of the snapshot</param>
/// <returns>The path of the snapshot root</returns>
/// <exception cref="System.Exception"/>
public static Path CreateSnapshot(DistributedFileSystem hdfs, Path snapshotRoot,
string snapshotName)
{
Log.Info("createSnapshot " + snapshotName + " for " + snapshotRoot);
NUnit.Framework.Assert.IsTrue(hdfs.Exists(snapshotRoot));
hdfs.AllowSnapshot(snapshotRoot);
hdfs.CreateSnapshot(snapshotRoot, snapshotName);
// set quota to a large value for testing counts
hdfs.SetQuota(snapshotRoot, long.MaxValue - 1, long.MaxValue - 1);
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.SnapshotTestHelper.GetSnapshotRoot
(snapshotRoot, snapshotName));
}
/// <summary>Test listing snapshots under a snapshottable directory</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshots()
{
Path snapshotsPath = new Path(dir, ".snapshot");
FileStatus[] stats = null;
// special case: snapshots of root
stats = hdfs.ListStatus(new Path("/.snapshot"));
// should be 0 since root's snapshot quota is 0
NUnit.Framework.Assert.AreEqual(0, stats.Length);
// list before set dir as snapshottable
try
{
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.Fail("expect SnapshotException");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Directory is not a snapshottable directory: "
+ dir.ToString(), e);
}
// list before creating snapshots
hdfs.AllowSnapshot(dir);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(0, stats.Length);
// list while creating snapshots
int snapshotNum = 5;
for (int sNum = 0; sNum < snapshotNum; sNum++)
{
hdfs.CreateSnapshot(dir, "s_" + sNum);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(sNum + 1, stats.Length);
for (int i = 0; i <= sNum; i++)
{
NUnit.Framework.Assert.AreEqual("s_" + i, stats[i].GetPath().GetName());
}
}
// list while deleting snapshots
for (int sNum_1 = snapshotNum - 1; sNum_1 > 0; sNum_1--)
{
hdfs.DeleteSnapshot(dir, "s_" + sNum_1);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(sNum_1, stats.Length);
for (int i = 0; i < sNum_1; i++)
{
NUnit.Framework.Assert.AreEqual("s_" + i, stats[i].GetPath().GetName());
}
}
// remove the last snapshot
hdfs.DeleteSnapshot(dir, "s_0");
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(0, stats.Length);
}
/*
* Try to read the files inside snapshot but deleted in original place after
* restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
hdfs.Mkdirs(foo);
hdfs.AllowSnapshot(foo);
Path bar = new Path("/foo/bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
string snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotPath
(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(snapshotPath));
}
public virtual void TestSnapshotStatsMXBeanInfo()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
string pathName = "/snapshot";
Path path = new Path(pathName);
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
SnapshotManager sm = cluster.GetNamesystem().GetSnapshotManager();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.GetFileSystem();
dfs.Mkdirs(path);
dfs.AllowSnapshot(path);
dfs.CreateSnapshot(path);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo"
);
CompositeData[] directories = (CompositeData[])mbs.GetAttribute(mxbeanName, "SnapshottableDirectories"
);
int numDirectories = Sharpen.Runtime.GetArrayLength(directories);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshottableDirs(), numDirectories);
CompositeData[] snapshots = (CompositeData[])mbs.GetAttribute(mxbeanName, "Snapshots"
);
int numSnapshots = Sharpen.Runtime.GetArrayLength(snapshots);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshots(), numSnapshots);
CompositeData d = (CompositeData)Sharpen.Runtime.GetArrayValue(directories, 0);
CompositeData s = (CompositeData)Sharpen.Runtime.GetArrayValue(snapshots, 0);
NUnit.Framework.Assert.IsTrue(((string)d.Get("path")).Contains(pathName));
NUnit.Framework.Assert.IsTrue(((string)s.Get("snapshotDirectory")).Contains(pathName
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void DoWriteAndAbort(DistributedFileSystem fs, Path path)
{
fs.Mkdirs(path);
fs.AllowSnapshot(path);
DFSTestUtil.CreateFile(fs, new Path("/test/test1"), 100, (short)2, 100024L);
DFSTestUtil.CreateFile(fs, new Path("/test/test2"), 100, (short)2, 100024L);
Path file = new Path("/test/test/test2");
FSDataOutputStream @out = fs.Create(file);
for (int i = 0; i < 2; i++)
{
long count = 0;
while (count < 1048576)
{
@out.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)@out.GetWrappedStream());
Path file2 = new Path("/test/test/test3");
FSDataOutputStream out2 = fs.Create(file2);
for (int i_1 = 0; i_1 < 2; i_1++)
{
long count = 0;
while (count < 1048576)
{
out2.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)out2.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)out2.GetWrappedStream());
fs.CreateSnapshot(path, "s1");
}
/// <summary>for snapshot file.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotPathINodes()
{
// Create a snapshot for the dir, and check the inodes for the path
// pointing to a snapshot file
hdfs.AllowSnapshot(sub1);
hdfs.CreateSnapshot(sub1, "s1");
// The path when accessing the snapshot file of file1 is
// /TestSnapshot/sub1/.snapshot/s1/file1
string snapshotPath = sub1.ToString() + "/.snapshot/s1/file1";
string[] names = INode.GetPathNames(snapshotPath);
byte[][] components = INode.GetPathComponents(names);
INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false);
// Length of inodes should be (components.length - 1), since we will ignore
// ".snapshot"
NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1);
// SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s1, file1}
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = GetSnapshot(nodesInPath
, "s1", 3);
AssertSnapshot(nodesInPath, true, snapshot, 3);
// Check the INode for file1 (snapshot file)
INode snapshotFileNode = nodesInPath.GetLastINode();
AssertINodeFile(snapshotFileNode, file1);
NUnit.Framework.Assert.IsTrue(snapshotFileNode.GetParent().IsWithSnapshot());
// Call getExistingPathINodes and request only one INode.
nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false);
NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1);
AssertSnapshot(nodesInPath, true, snapshot, 3);
// Check the INode for file1 (snapshot file)
AssertINodeFile(nodesInPath.GetLastINode(), file1);
// Resolve the path "/TestSnapshot/sub1/.snapshot"
string dotSnapshotPath = sub1.ToString() + "/.snapshot";
names = INode.GetPathNames(dotSnapshotPath);
components = INode.GetPathComponents(names);
nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false);
// The number of INodes returned should still be components.length
// since we put a null in the inode array for ".snapshot"
NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length);
// No SnapshotRoot dir is included in the resolved inodes
AssertSnapshot(nodesInPath, true, snapshot, -1);
// The last INode should be null, the last but 1 should be sub1
NUnit.Framework.Assert.IsNull(nodesInPath.GetLastINode());
NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(-2).GetFullPathName(), sub1.
ToString());
NUnit.Framework.Assert.IsTrue(nodesInPath.GetINode(-2).IsDirectory());
string[] invalidPathComponent = new string[] { "invalidDir", "foo", ".snapshot",
"bar" };
Path invalidPath = new Path(invalidPathComponent[0]);
for (int i = 1; i < invalidPathComponent.Length; i++)
{
invalidPath = new Path(invalidPath, invalidPathComponent[i]);
try
{
hdfs.GetFileStatus(invalidPath);
NUnit.Framework.Assert.Fail();
}
catch (FileNotFoundException fnfe)
{
System.Console.Out.WriteLine("The exception is expected: " + fnfe);
}
}
hdfs.DeleteSnapshot(sub1, "s1");
hdfs.DisallowSnapshot(sub1);
}
/// <summary>Test listing all the snapshottable directories</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// Initially there is no snapshottable directories in the system
SnapshottableDirectoryStatus[] dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make root as snapshottable
Path root = new Path("/");
hdfs.AllowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(string.Empty, dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(root, dirs[0].GetFullPath());
// Make root non-snaphsottable
hdfs.DisallowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make dir1 as snapshottable
hdfs.AllowSnapshot(dir1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// There is no snapshot for dir1 yet
NUnit.Framework.Assert.AreEqual(0, dirs[0].GetSnapshotNumber());
// Make dir2 as snapshottable
hdfs.AllowSnapshot(dir2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[1].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
// There is no snapshot for dir2 yet
NUnit.Framework.Assert.AreEqual(0, dirs[1].GetSnapshotNumber());
// Create dir3
Path dir3 = new Path("/TestSnapshot3");
hdfs.Mkdirs(dir3);
// Rename dir3 to dir2
hdfs.Rename(dir3, dir2, Options.Rename.Overwrite);
// Now we only have one snapshottable dir: dir1
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// Make dir2 snapshottable again
hdfs.AllowSnapshot(dir2);
// Create a snapshot for dir2
hdfs.CreateSnapshot(dir2, "s1");
hdfs.CreateSnapshot(dir2, "s2");
dirs = hdfs.GetSnapshottableDirListing();
// There are now 2 snapshots for dir2
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(2, dirs[1].GetSnapshotNumber());
// Create sub-dirs under dir1
Path sub1 = new Path(dir1, "sub1");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(dir1, "sub2");
Path file2 = new Path(sub2, "file2");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, seed);
// Make sub1 and sub2 snapshottable
hdfs.AllowSnapshot(sub1);
hdfs.AllowSnapshot(sub2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(4, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub1, dirs[2].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[3].GetFullPath());
// reset sub1
hdfs.DisallowSnapshot(sub1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(3, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[2].GetFullPath());
// Remove dir1, both dir1 and sub2 will be removed
hdfs.Delete(dir1, true);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[0].GetFullPath());
}
/// <summary>
/// Test that we cannot read a file beyond its snapshot length
/// when accessing it via a snapshot path.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotfileLength()
{
hdfs.Mkdirs(sub);
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
int origLen = Blocksize + 1;
int toAppend = Blocksize;
FSDataInputStream fis = null;
FileStatus fileStatus = null;
// Create and write a file.
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed);
DFSTestUtil.AppendFile(hdfs, file1, origLen);
// Create a snapshot on the parent directory.
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1);
Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
// Append to the file.
FSDataOutputStream @out = hdfs.Append(file1);
// Nothing has been appended yet. All checksums should still be equal.
Assert.AssertThat("file and snapshot checksums (open for append) are not equal",
hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
try
{
AppendTestUtil.Write(@out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is
(origLen));
// Verify that checksum didn't change
Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
}
finally
{
@out.Close();
}
Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs
.GetFileChecksum(file1), CoreMatchers.Not(snapChksum1));
Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend));
fis = hdfs.Open(file1);
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend));
fis.Close();
// Try to open the file via its snapshot path.
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen));
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen));
fis.Close();
byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is
(origLen));
}
/// <summary>
/// Create a snapshot for /test/foo and create another snapshot for
/// /test/foo/bar.
/// </summary>
/// <remarks>
/// Create a snapshot for /test/foo and create another snapshot for
/// /test/foo/bar. Files created before the snapshots should appear in both
/// snapshots and the files created after the snapshots should not appear in
/// any of the snapshots.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestNestedSnapshots()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
Path foo = new Path("/testNestedSnapshots/foo");
Path bar = new Path(foo, "bar");
Path file1 = new Path(bar, "file1");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
Print("create file " + file1);
string s1name = "foo-s1";
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, s1name);
hdfs.AllowSnapshot(foo);
Print("allow snapshot " + foo);
hdfs.CreateSnapshot(foo, s1name);
Print("create snapshot " + s1name);
string s2name = "bar-s2";
Path s2path = SnapshotTestHelper.GetSnapshotRoot(bar, s2name);
hdfs.AllowSnapshot(bar);
Print("allow snapshot " + bar);
hdfs.CreateSnapshot(bar, s2name);
Print("create snapshot " + s2name);
Path file2 = new Path(bar, "file2");
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, Seed);
Print("create file " + file2);
AssertFile(s1path, s2path, file1, true, true, true);
AssertFile(s1path, s2path, file2, true, false, false);
//test root
string rootStr = "/";
Path rootPath = new Path(rootStr);
hdfs.AllowSnapshot(rootPath);
Print("allow snapshot " + rootStr);
Path rootSnapshot = hdfs.CreateSnapshot(rootPath);
Print("create snapshot " + rootSnapshot);
hdfs.DeleteSnapshot(rootPath, rootSnapshot.GetName());
Print("delete snapshot " + rootSnapshot);
hdfs.DisallowSnapshot(rootPath);
Print("disallow snapshot " + rootStr);
//change foo to non-snapshottable
hdfs.DeleteSnapshot(foo, s1name);
hdfs.DisallowSnapshot(foo);
//test disallow nested snapshots
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(false);
try
{
hdfs.AllowSnapshot(rootPath);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "subdirectory");
}
try
{
hdfs.AllowSnapshot(foo);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "subdirectory");
}
Path sub1Bar = new Path(bar, "sub1");
Path sub2Bar = new Path(sub1Bar, "sub2");
hdfs.Mkdirs(sub2Bar);
try
{
hdfs.AllowSnapshot(sub1Bar);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "ancestor");
}
try
{
hdfs.AllowSnapshot(sub2Bar);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "ancestor");
}
}