public virtual void TestSnapshotOnRoot()
{
Path root = new Path("/");
hdfs.AllowSnapshot(root);
hdfs.CreateSnapshot(root, "s1");
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
INodeDirectory rootNode = fsn.dir.GetINode4Write(root.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue("The children list of root should be empty", rootNode
.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId
).IsEmpty());
// one snapshot on root: s1
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = rootNode.GetDiffs().
AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s1 = rootNode.GetSnapshot
(DFSUtil.String2Bytes("s1"));
NUnit.Framework.Assert.AreEqual(s1.GetId(), diffList[0].GetSnapshotId());
// check SnapshotManager's snapshottable directory list
NUnit.Framework.Assert.AreEqual(1, fsn.GetSnapshotManager().GetNumSnapshottableDirs
());
SnapshottableDirectoryStatus[] sdirs = fsn.GetSnapshotManager().GetSnapshottableDirListing
(null);
NUnit.Framework.Assert.AreEqual(root, sdirs[0].GetFullPath());
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
}
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotWithInvalidName()
{
Path file1 = new Path(dir1, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(dir1);
try
{
hdfs.CreateSnapshot(dir1, snapshot1);
}
catch (RemoteException)
{
}
}
public virtual void TestWebHdfsRenameSnapshot()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
Path foo = new Path("/foo");
dfs.Mkdirs(foo);
dfs.AllowSnapshot(foo);
webHdfs.CreateSnapshot(foo, "s1");
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path));
// rename s1 to s2
webHdfs.RenameSnapshot(foo, "s1", "s2");
NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s1path));
Path s2path = SnapshotTestHelper.GetSnapshotRoot(foo, "s2");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s2path));
webHdfs.DeleteSnapshot(foo, "s2");
NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s2path));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void SetUp()
{
fs.Mkdirs(new Path("/sub1"));
fs.AllowSnapshot(new Path("/sub1"));
fs.Mkdirs(new Path("/sub1/sub1sub1"));
fs.Mkdirs(new Path("/sub1/sub1sub2"));
}
/// <summary>
/// Create/modify/delete files under a given directory, also create snapshots
/// of directories.
/// </summary>
/// <exception cref="System.Exception"/>
private void ModifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
{
Path file10 = new Path(modifyDir, "file10");
Path file11 = new Path(modifyDir, "file11");
Path file12 = new Path(modifyDir, "file12");
Path file13 = new Path(modifyDir, "file13");
Path link13 = new Path(modifyDir, "link13");
Path file14 = new Path(modifyDir, "file14");
Path file15 = new Path(modifyDir, "file15");
DFSTestUtil.CreateFile(hdfs, file10, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file11, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file12, Blocksize, Replication1, seed);
DFSTestUtil.CreateFile(hdfs, file13, Blocksize, Replication1, seed);
// create link13
hdfs.CreateSymlink(file13, link13, false);
// create snapshot
foreach (Path snapshotDir in snapshotDirs)
{
hdfs.AllowSnapshot(snapshotDir);
hdfs.CreateSnapshot(snapshotDir, GenSnapshotName(snapshotDir));
}
// delete file11
hdfs.Delete(file11, true);
// modify file12
hdfs.SetReplication(file12, Replication);
// modify file13
hdfs.SetReplication(file13, Replication);
// delete link13
hdfs.Delete(link13, false);
// create file14
DFSTestUtil.CreateFile(hdfs, file14, Blocksize, Replication, seed);
// create file15
DFSTestUtil.CreateFile(hdfs, file15, Blocksize, Replication, seed);
// create snapshot
foreach (Path snapshotDir_1 in snapshotDirs)
{
hdfs.CreateSnapshot(snapshotDir_1, GenSnapshotName(snapshotDir_1));
}
// create file11 again
DFSTestUtil.CreateFile(hdfs, file11, Blocksize, Replication, seed);
// delete file12
hdfs.Delete(file12, true);
// modify file13
hdfs.SetReplication(file13, (short)(Replication - 2));
// create link13 again
hdfs.CreateSymlink(file13, link13, false);
// delete file14
hdfs.Delete(file14, true);
// modify file15
hdfs.SetReplication(file15, (short)(Replication - 1));
// create snapshot
foreach (Path snapshotDir_2 in snapshotDirs)
{
hdfs.CreateSnapshot(snapshotDir_2, GenSnapshotName(snapshotDir_2));
}
// modify file10
hdfs.SetReplication(file10, (short)(Replication + 1));
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
/// <summary>Create snapshot for a dir using a given snapshot name</summary>
/// <param name="hdfs">DistributedFileSystem instance</param>
/// <param name="snapshotRoot">The dir to be snapshotted</param>
/// <param name="snapshotName">The name of the snapshot</param>
/// <returns>The path of the snapshot root</returns>
/// <exception cref="System.Exception"/>
public static Path CreateSnapshot(DistributedFileSystem hdfs, Path snapshotRoot,
string snapshotName)
{
Log.Info("createSnapshot " + snapshotName + " for " + snapshotRoot);
NUnit.Framework.Assert.IsTrue(hdfs.Exists(snapshotRoot));
hdfs.AllowSnapshot(snapshotRoot);
hdfs.CreateSnapshot(snapshotRoot, snapshotName);
// set quota to a large value for testing counts
hdfs.SetQuota(snapshotRoot, long.MaxValue - 1, long.MaxValue - 1);
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.SnapshotTestHelper.GetSnapshotRoot
(snapshotRoot, snapshotName));
}
/// <summary>Test allow-snapshot operation.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestAllowSnapshot()
{
string pathStr = sub1.ToString();
INode before = fsdir.GetINode(pathStr);
// Before a directory is snapshottable
NUnit.Framework.Assert.IsFalse(before.AsDirectory().IsSnapshottable());
// After a directory is snapshottable
Path path = new Path(pathStr);
hdfs.AllowSnapshot(path);
{
INode after = fsdir.GetINode(pathStr);
NUnit.Framework.Assert.IsTrue(after.AsDirectory().IsSnapshottable());
}
hdfs.DisallowSnapshot(path);
{
INode after = fsdir.GetINode(pathStr);
NUnit.Framework.Assert.IsFalse(after.AsDirectory().IsSnapshottable());
}
}
/// <summary>Test listing snapshots under a snapshottable directory</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshots()
{
Path snapshotsPath = new Path(dir, ".snapshot");
FileStatus[] stats = null;
// special case: snapshots of root
stats = hdfs.ListStatus(new Path("/.snapshot"));
// should be 0 since root's snapshot quota is 0
NUnit.Framework.Assert.AreEqual(0, stats.Length);
// list before set dir as snapshottable
try
{
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.Fail("expect SnapshotException");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Directory is not a snapshottable directory: "
+ dir.ToString(), e);
}
// list before creating snapshots
hdfs.AllowSnapshot(dir);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(0, stats.Length);
// list while creating snapshots
int snapshotNum = 5;
for (int sNum = 0; sNum < snapshotNum; sNum++)
{
hdfs.CreateSnapshot(dir, "s_" + sNum);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(sNum + 1, stats.Length);
for (int i = 0; i <= sNum; i++)
{
NUnit.Framework.Assert.AreEqual("s_" + i, stats[i].GetPath().GetName());
}
}
// list while deleting snapshots
for (int sNum_1 = snapshotNum - 1; sNum_1 > 0; sNum_1--)
{
hdfs.DeleteSnapshot(dir, "s_" + sNum_1);
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(sNum_1, stats.Length);
for (int i = 0; i < sNum_1; i++)
{
NUnit.Framework.Assert.AreEqual("s_" + i, stats[i].GetPath().GetName());
}
}
// remove the last snapshot
hdfs.DeleteSnapshot(dir, "s_0");
stats = hdfs.ListStatus(snapshotsPath);
NUnit.Framework.Assert.AreEqual(0, stats.Length);
}
public virtual void TestSnapshottableDirs()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 0, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 0L, MetricsAsserts.GetMetrics(NnMetrics
));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 0L, MetricsAsserts.GetMetrics
(NnMetrics));
// Allow snapshots for directories, and check the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
Path sub2 = new Path(dir, "sub2");
Path file = new Path(sub2, "file");
DFSTestUtil.CreateFile(hdfs, file, 1024, Replication, seed);
hdfs.AllowSnapshot(sub2);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 2L, MetricsAsserts.GetMetrics(NnMetrics
));
Path subsub1 = new Path(sub1, "sub1sub1");
Path subfile = new Path(subsub1, "file");
DFSTestUtil.CreateFile(hdfs, subfile, 1024, Replication, seed);
hdfs.AllowSnapshot(subsub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 3L, MetricsAsserts.GetMetrics(NnMetrics
));
// Set an already snapshottable directory to snapshottable, should not
// change the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
// But the number of allowSnapshot operations still increases
MetricsAsserts.AssertCounter("AllowSnapshotOps", 4L, MetricsAsserts.GetMetrics(NnMetrics
));
// Disallow the snapshot for snapshottable directories, then check the
// metrics again
hdfs.DisallowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
// delete subsub1, snapshottable directories should be 1
hdfs.Delete(subsub1, true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
// list all the snapshottable directories
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
MetricsAsserts.AssertCounter("ListSnapshottableDirOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
}
/*
* Try to read the files inside snapshot but deleted in original place after
* restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
hdfs.Mkdirs(foo);
hdfs.AllowSnapshot(foo);
Path bar = new Path("/foo/bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
string snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotPath
(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(snapshotPath));
}
public virtual void TestWebHdfsCreateSnapshot()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
Path foo = new Path("/foo");
dfs.Mkdirs(foo);
try
{
webHdfs.CreateSnapshot(foo);
NUnit.Framework.Assert.Fail("Cannot create snapshot on a non-snapshottable directory"
);
}
catch (Exception e)
{
GenericTestUtils.AssertExceptionContains("Directory is not a snapshottable directory"
, e);
}
// allow snapshots on /foo
dfs.AllowSnapshot(foo);
// create snapshots on foo using WebHdfs
webHdfs.CreateSnapshot(foo, "s1");
// create snapshot without specifying name
Path spath = webHdfs.CreateSnapshot(foo, null);
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(spath));
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSnapshotStatsMXBeanInfo()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
string pathName = "/snapshot";
Path path = new Path(pathName);
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
SnapshotManager sm = cluster.GetNamesystem().GetSnapshotManager();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.GetFileSystem();
dfs.Mkdirs(path);
dfs.AllowSnapshot(path);
dfs.CreateSnapshot(path);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo"
);
CompositeData[] directories = (CompositeData[])mbs.GetAttribute(mxbeanName, "SnapshottableDirectories"
);
int numDirectories = Sharpen.Runtime.GetArrayLength(directories);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshottableDirs(), numDirectories);
CompositeData[] snapshots = (CompositeData[])mbs.GetAttribute(mxbeanName, "Snapshots"
);
int numSnapshots = Sharpen.Runtime.GetArrayLength(snapshots);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshots(), numSnapshots);
CompositeData d = (CompositeData)Sharpen.Runtime.GetArrayValue(directories, 0);
CompositeData s = (CompositeData)Sharpen.Runtime.GetArrayValue(snapshots, 0);
NUnit.Framework.Assert.IsTrue(((string)d.Get("path")).Contains(pathName));
NUnit.Framework.Assert.IsTrue(((string)s.Get("snapshotDirectory")).Contains(pathName
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void DoWriteAndAbort(DistributedFileSystem fs, Path path)
{
fs.Mkdirs(path);
fs.AllowSnapshot(path);
DFSTestUtil.CreateFile(fs, new Path("/test/test1"), 100, (short)2, 100024L);
DFSTestUtil.CreateFile(fs, new Path("/test/test2"), 100, (short)2, 100024L);
Path file = new Path("/test/test/test2");
FSDataOutputStream @out = fs.Create(file);
for (int i = 0; i < 2; i++)
{
long count = 0;
while (count < 1048576)
{
@out.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)@out.GetWrappedStream());
Path file2 = new Path("/test/test/test3");
FSDataOutputStream out2 = fs.Create(file2);
for (int i_1 = 0; i_1 < 2; i_1++)
{
long count = 0;
while (count < 1048576)
{
out2.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)out2.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)out2.GetWrappedStream());
fs.CreateSnapshot(path, "s1");
}
/// <summary>Allow snapshot on a directory.</summary>
/// <param name="path">The path of the directory where snapshots will be taken.</param>
/// <exception cref="System.IO.IOException"/>
public virtual void AllowSnapshot(Path path)
{
dfs.AllowSnapshot(path);
}
public static void CreateOriginalFSImage()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeKey, 10000);
conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenRenewIntervalKey, 5000);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//"
+ "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
// Create a reasonable namespace
for (int i = 0; i < NumDirs; i++)
{
Path dir = new Path("/dir" + i);
hdfs.Mkdirs(dir);
writtenFiles[dir.ToString()] = PathToFileEntry(hdfs, dir.ToString());
for (int j = 0; j < FilesPerDir; j++)
{
Path file = new Path(dir, "file" + j);
FSDataOutputStream o = hdfs.Create(file);
o.Write(23);
o.Close();
writtenFiles[file.ToString()] = PathToFileEntry(hdfs, file.ToString());
}
}
// Create an empty directory
Path emptydir = new Path("/emptydir");
hdfs.Mkdirs(emptydir);
writtenFiles[emptydir.ToString()] = hdfs.GetFileStatus(emptydir);
//Create a directory whose name should be escaped in XML
Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
hdfs.Mkdirs(invalidXMLDir);
// Get delegation tokens so we log the delegation token op
Org.Apache.Hadoop.Security.Token.Token <object>[] delegationTokens = hdfs.AddDelegationTokens
(TestRenewer, null);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> t in delegationTokens)
{
Log.Debug("got token " + t);
}
Path snapshot = new Path("/snapshot");
hdfs.Mkdirs(snapshot);
hdfs.AllowSnapshot(snapshot);
hdfs.Mkdirs(new Path("/snapshot/1"));
hdfs.Delete(snapshot, true);
// Set XAttrs so the fsimage contains XAttr ops
Path xattr = new Path("/xattr");
hdfs.Mkdirs(xattr);
hdfs.SetXAttr(xattr, "user.a1", new byte[] { unchecked ((int)(0x31)), unchecked ((int
)(0x32)), unchecked ((int)(0x33)) });
hdfs.SetXAttr(xattr, "user.a2", new byte[] { unchecked ((int)(0x37)), unchecked ((int
)(0x38)), unchecked ((int)(0x39)) });
// OIV should be able to handle empty value XAttrs
hdfs.SetXAttr(xattr, "user.a3", null);
writtenFiles[xattr.ToString()] = hdfs.GetFileStatus(xattr);
// Write results to the fsimage file
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false);
hdfs.SaveNamespace();
// Determine location of fsimage file
originalFsimage = FSImageTestUtil.FindLatestImageFile(FSImageTestUtil.GetFSImage(
cluster.GetNameNode()).GetStorage().GetStorageDir(0));
if (originalFsimage == null)
{
throw new RuntimeException("Didn't generate or can't find fsimage");
}
Log.Debug("original FS image file is " + originalFsimage);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Create a snapshot for /test/foo and create another snapshot for
/// /test/foo/bar.
/// </summary>
/// <remarks>
/// Create a snapshot for /test/foo and create another snapshot for
/// /test/foo/bar. Files created before the snapshots should appear in both
/// snapshots and the files created after the snapshots should not appear in
/// any of the snapshots.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestNestedSnapshots()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
Path foo = new Path("/testNestedSnapshots/foo");
Path bar = new Path(foo, "bar");
Path file1 = new Path(bar, "file1");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
Print("create file " + file1);
string s1name = "foo-s1";
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, s1name);
hdfs.AllowSnapshot(foo);
Print("allow snapshot " + foo);
hdfs.CreateSnapshot(foo, s1name);
Print("create snapshot " + s1name);
string s2name = "bar-s2";
Path s2path = SnapshotTestHelper.GetSnapshotRoot(bar, s2name);
hdfs.AllowSnapshot(bar);
Print("allow snapshot " + bar);
hdfs.CreateSnapshot(bar, s2name);
Print("create snapshot " + s2name);
Path file2 = new Path(bar, "file2");
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, Seed);
Print("create file " + file2);
AssertFile(s1path, s2path, file1, true, true, true);
AssertFile(s1path, s2path, file2, true, false, false);
//test root
string rootStr = "/";
Path rootPath = new Path(rootStr);
hdfs.AllowSnapshot(rootPath);
Print("allow snapshot " + rootStr);
Path rootSnapshot = hdfs.CreateSnapshot(rootPath);
Print("create snapshot " + rootSnapshot);
hdfs.DeleteSnapshot(rootPath, rootSnapshot.GetName());
Print("delete snapshot " + rootSnapshot);
hdfs.DisallowSnapshot(rootPath);
Print("disallow snapshot " + rootStr);
//change foo to non-snapshottable
hdfs.DeleteSnapshot(foo, s1name);
hdfs.DisallowSnapshot(foo);
//test disallow nested snapshots
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(false);
try
{
hdfs.AllowSnapshot(rootPath);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "subdirectory");
}
try
{
hdfs.AllowSnapshot(foo);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "subdirectory");
}
Path sub1Bar = new Path(bar, "sub1");
Path sub2Bar = new Path(sub1Bar, "sub2");
hdfs.Mkdirs(sub2Bar);
try
{
hdfs.AllowSnapshot(sub1Bar);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "ancestor");
}
try
{
hdfs.AllowSnapshot(sub2Bar);
NUnit.Framework.Assert.Fail();
}
catch (SnapshotException se)
{
AssertNestedSnapshotException(se, "ancestor");
}
}
/// <summary>Test listing all the snapshottable directories</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// Initially there is no snapshottable directories in the system
SnapshottableDirectoryStatus[] dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make root as snapshottable
Path root = new Path("/");
hdfs.AllowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(string.Empty, dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(root, dirs[0].GetFullPath());
// Make root non-snaphsottable
hdfs.DisallowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make dir1 as snapshottable
hdfs.AllowSnapshot(dir1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// There is no snapshot for dir1 yet
NUnit.Framework.Assert.AreEqual(0, dirs[0].GetSnapshotNumber());
// Make dir2 as snapshottable
hdfs.AllowSnapshot(dir2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[1].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
// There is no snapshot for dir2 yet
NUnit.Framework.Assert.AreEqual(0, dirs[1].GetSnapshotNumber());
// Create dir3
Path dir3 = new Path("/TestSnapshot3");
hdfs.Mkdirs(dir3);
// Rename dir3 to dir2
hdfs.Rename(dir3, dir2, Options.Rename.Overwrite);
// Now we only have one snapshottable dir: dir1
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// Make dir2 snapshottable again
hdfs.AllowSnapshot(dir2);
// Create a snapshot for dir2
hdfs.CreateSnapshot(dir2, "s1");
hdfs.CreateSnapshot(dir2, "s2");
dirs = hdfs.GetSnapshottableDirListing();
// There are now 2 snapshots for dir2
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(2, dirs[1].GetSnapshotNumber());
// Create sub-dirs under dir1
Path sub1 = new Path(dir1, "sub1");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(dir1, "sub2");
Path file2 = new Path(sub2, "file2");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, seed);
// Make sub1 and sub2 snapshottable
hdfs.AllowSnapshot(sub1);
hdfs.AllowSnapshot(sub2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(4, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub1, dirs[2].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[3].GetFullPath());
// reset sub1
hdfs.DisallowSnapshot(sub1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(3, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[2].GetFullPath());
// Remove dir1, both dir1 and sub2 will be removed
hdfs.Delete(dir1, true);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[0].GetFullPath());
}
/// <summary>
/// Test that we cannot read a file beyond its snapshot length
/// when accessing it via a snapshot path.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotfileLength()
{
hdfs.Mkdirs(sub);
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
int origLen = Blocksize + 1;
int toAppend = Blocksize;
FSDataInputStream fis = null;
FileStatus fileStatus = null;
// Create and write a file.
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed);
DFSTestUtil.AppendFile(hdfs, file1, origLen);
// Create a snapshot on the parent directory.
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1);
Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
// Append to the file.
FSDataOutputStream @out = hdfs.Append(file1);
// Nothing has been appended yet. All checksums should still be equal.
Assert.AssertThat("file and snapshot checksums (open for append) are not equal",
hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
try
{
AppendTestUtil.Write(@out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is
(origLen));
// Verify that checksum didn't change
Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
}
finally
{
@out.Close();
}
Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs
.GetFileChecksum(file1), CoreMatchers.Not(snapChksum1));
Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend));
fis = hdfs.Open(file1);
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend));
fis.Close();
// Try to open the file via its snapshot path.
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen));
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen));
fis.Close();
byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is
(origLen));
}