public virtual void TestDeletionOfLaterBlocksWithZeroSizeFirstBlock()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
byte[] testData = Sharpen.Runtime.GetBytesForString("foo bar baz");
// Create a zero-length file.
DFSTestUtil.CreateFile(hdfs, bar, 0, Replication, 0L);
NUnit.Framework.Assert.AreEqual(0, fsdir.GetINode4Write(bar.ToString()).AsFile().
GetBlocks().Length);
// Create a snapshot that includes that file.
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s0");
// Extend that file.
FSDataOutputStream @out = hdfs.Append(bar);
@out.Write(testData);
@out.Close();
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(testData.Length, blks[0].GetNumBytes());
// Delete the file.
hdfs.Delete(bar, true);
// Now make sure that the NN can still save an fsimage successfully.
cluster.GetNameNode().GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter
, false);
cluster.GetNameNode().GetRpcServer().SaveNamespace();
}
public virtual void TestSetXAttrSnapshotPath()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
exception.Expect(typeof(SnapshotAccessControlException));
hdfs.SetXAttr(snapshotPath, name1, value1);
}
/// <summary>
/// Test replication for a file with snapshots, also including the scenario
/// where the original file is deleted
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestReplicationAfterDeletion()
{
// Create file1, set its replication to 3
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>();
// Take 3 snapshots of sub1
for (int i = 1; i <= 3; i++)
{
Path root = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + i);
Path ssFile = new Path(root, file1.GetName());
snapshotRepMap[ssFile] = Replication;
}
// Check replication
CheckFileReplication(file1, Replication, Replication);
CheckSnapshotFileReplication(file1, snapshotRepMap, Replication);
// Delete file1
hdfs.Delete(file1, true);
// Check replication of snapshots
foreach (Path ss in snapshotRepMap.Keys)
{
INodeFile ssInode = GetINodeFile(ss);
// The replication number derived from the
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
NUnit.Framework.Assert.AreEqual(Replication, ssInode.GetBlockReplication());
// Also check the number derived from INodeFile#getFileReplication
NUnit.Framework.Assert.AreEqual(snapshotRepMap[ss], ssInode.GetFileReplication());
}
}
public virtual void TestDeletionWithZeroSizeBlock2()
{
Path foo = new Path("/foo");
Path subDir = new Path(foo, "sub");
Path bar = new Path(subDir, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, 0L);
hdfs.Append(bar);
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
ExtendedBlock previous = new ExtendedBlock(fsn.GetBlockPoolId(), blks[0]);
cluster.GetNameNodeRpc().AddBlock(bar.ToString(), hdfs.GetClient().GetClientName(
), previous, null, barNode.GetId(), null);
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s1");
barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(0, blks[1].GetNumBytes());
hdfs.Delete(subDir, true);
Path sbar = SnapshotTestHelper.GetSnapshotPath(foo, "s1", "sub/bar");
barNode = fsdir.GetINode(sbar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
}
/// <summary>Tests modifying xattrs on a directory that has been snapshotted</summary>
/// <exception cref="System.Exception"/>
public virtual void TestModifyReadsCurrentState()
{
// Init
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
// Verify that current path reflects xattrs, snapshot doesn't
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
// Modify each xattr and make sure it's reflected
hdfs.SetXAttr(path, name1, value2, EnumSet.Of(XAttrSetFlag.Replace));
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value2, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
hdfs.SetXAttr(path, name2, value1, EnumSet.Of(XAttrSetFlag.Replace));
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value2, xattrs[name1]);
Assert.AssertArrayEquals(value1, xattrs[name2]);
// Paranoia checks
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
hdfs.RemoveXAttr(path, name1);
hdfs.RemoveXAttr(path, name2);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
}
/// <summary>Test replication number calculation for a file with snapshots.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestReplicationWithSnapshot()
{
short fileRep = 1;
// Create file1, set its replication to 1
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, fileRep, seed);
IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>();
// Change replication factor from 1 to 5. In the meanwhile, keep taking
// snapshots for sub1
for (; fileRep < Numdatanode;)
{
// Create snapshot for sub1
Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + fileRep);
Path snapshot = new Path(snapshotRoot, file1.GetName());
// Check the replication stored in the INode of the snapshot of file1
NUnit.Framework.Assert.AreEqual(fileRep, GetINodeFile(snapshot).GetFileReplication
());
snapshotRepMap[snapshot] = fileRep;
// Increase the replication factor by 1
hdfs.SetReplication(file1, ++fileRep);
// Check the replication for file1
CheckFileReplication(file1, fileRep, fileRep);
// Also check the replication for all the prior snapshots of file1
CheckSnapshotFileReplication(file1, snapshotRepMap, fileRep);
}
// Change replication factor back to 3.
hdfs.SetReplication(file1, Replication);
// Check the replication for file1
// Currently the max replication among snapshots should be 4
CheckFileReplication(file1, Replication, (short)(Numdatanode - 1));
// Also check the replication for all the prior snapshots of file1.
// Currently the max replication among snapshots should be 4
CheckSnapshotFileReplication(file1, snapshotRepMap, (short)(Numdatanode - 1));
}
/// <summary>Test snapshot after file appending</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotAfterAppending()
{
Path file = new Path(dir, "file");
// 1. create snapshot --> create file --> append
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
// 2. create snapshot --> modify the file --> append
hdfs.CreateSnapshot(dir, "s1");
hdfs.SetReplication(file, (short)(Replication - 1));
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize());
// 3. create snapshot --> append
hdfs.CreateSnapshot(dir, "s2");
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize());
}
public TestINodeFileUnderConstructionWithSnapshot()
{
{
((Log4JLogger)INode.Log).GetLogger().SetLevel(Level.All);
SnapshotTestHelper.DisableLogs();
}
}
public virtual void TestDiffReportWithRenameOutside()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
// create snapshot on /dir1
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s0");
// move bar into dir1
Path newBar = new Path(dir1, "newBar");
hdfs.Rename(bar, newBar);
// move foo out of dir1 into dir2
Path newFoo = new Path(dir2, "new");
hdfs.Rename(foo, newFoo);
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s1");
VerifyDiffReport(dir1, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes(newBar.GetName())), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes
(foo.GetName())));
}
public virtual void TestDiffReportWithRename()
{
Path root = new Path("/");
Path sdir1 = new Path(root, "dir1");
Path sdir2 = new Path(root, "dir2");
Path foo = new Path(sdir1, "foo");
Path bar = new Path(foo, "bar");
hdfs.Mkdirs(bar);
hdfs.Mkdirs(sdir2);
// create snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
// /dir1/foo/bar -> /dir2/bar
Path bar2 = new Path(sdir2, "bar");
hdfs.Rename(bar, bar2);
// /dir1/foo -> /dir2/bar/foo
Path foo2 = new Path(bar2, "foo");
hdfs.Rename(foo, foo2);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
// let's delete /dir2 to make things more complicated
hdfs.Delete(sdir2, true);
VerifyDiffReport(root, "s1", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo"), DFSUtil.String2Bytes
("dir2/bar/foo")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes("dir1/foo/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo/bar"), DFSUtil
.String2Bytes("dir2/bar")));
}
public virtual void TestRenameWithIllegalName()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
string name1 = HdfsConstants.DotSnapshotDir;
try
{
hdfs.RenameSnapshot(sub1, "s1", name1);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given for rename"
);
}
catch (RemoteException e)
{
string errorMsg = "\"" + HdfsConstants.DotSnapshotDir + "\" is a reserved name.";
GenericTestUtils.AssertExceptionContains(errorMsg, e);
}
string errorMsg_1 = "Snapshot name cannot contain \"" + Path.Separator + "\"";
string[] badNames = new string[] { "foo" + Path.Separator, Path.Separator + "foo"
, Path.Separator, "foo" + Path.Separator + "bar" };
foreach (string badName in badNames)
{
try
{
hdfs.RenameSnapshot(sub1, "s1", badName);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given");
}
catch (RemoteException e)
{
GenericTestUtils.AssertExceptionContains(errorMsg_1, e);
}
}
}
public virtual void TestDiffReportWithRenameAndDelete()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
hdfs.Rename(fileInFoo, fileInBar, Options.Rename.Overwrite);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar/file")), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes
("dir1/foo/file"), DFSUtil.String2Bytes("dir2/bar/file")));
// delete bar
hdfs.Delete(bar, true);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
VerifyDiffReport(root, "s0", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir1/foo/file")));
}
/// <summary>Test FileStatus of snapshot file before/after rename</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotRename()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshot for sub1
Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
Path ssPath = new Path(snapshotRoot, file1.GetName());
NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath));
FileStatus statusBeforeRename = hdfs.GetFileStatus(ssPath);
// Rename the snapshot
hdfs.RenameSnapshot(sub1, "s1", "s2");
// <sub1>/.snapshot/s1/file1 should no longer exist
NUnit.Framework.Assert.IsFalse(hdfs.Exists(ssPath));
snapshotRoot = SnapshotTestHelper.GetSnapshotRoot(sub1, "s2");
ssPath = new Path(snapshotRoot, file1.GetName());
// Instead, <sub1>/.snapshot/s2/file1 should exist
NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath));
FileStatus statusAfterRename = hdfs.GetFileStatus(ssPath);
// FileStatus of the snapshot should not change except the path
NUnit.Framework.Assert.IsFalse(statusBeforeRename.Equals(statusAfterRename));
statusBeforeRename.SetPath(statusAfterRename.GetPath());
NUnit.Framework.Assert.AreEqual(statusBeforeRename.ToString(), statusAfterRename.
ToString());
}
public virtual void TestXAttrForSnapshotRootAfterRemove()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
// Both original and snapshot have same XAttrs.
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Original XAttrs have been removed, but snapshot still has old XAttrs.
hdfs.RemoveXAttr(path, name1);
hdfs.RemoveXAttr(path, name2);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
Restart(false);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
Restart(true);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
}
public virtual void TestDiffReportWithRenameAndSnapshotDeletion()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
// rename /foo to /foo2
Path foo2 = new Path(root, "foo2");
hdfs.Rename(foo, foo2);
// now /foo/bar becomes /foo2/bar
Path bar2 = new Path(foo2, "bar");
// delete snapshot s0 so that the withNameList inside of the WithCount node
// of foo becomes empty
hdfs.DeleteSnapshot(root, "s0");
// create snapshot s1 and rename bar again
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path bar3 = new Path(foo2, "bar-new");
hdfs.Rename(bar2, bar3);
// we always put modification on the file before rename
VerifyDiffReport(root, "s1", string.Empty, new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo2/bar"), DFSUtil.String2Bytes
("foo2/bar-new")));
}
/// <exception cref="System.Exception"/>
internal virtual void AssertINodeNullInSnapshots(Path path, params string[] snapshots
)
{
foreach (string s in snapshots)
{
AssertINodeNull(SnapshotTestHelper.GetSnapshotPath(path.GetParent(), s, path.GetName
()).ToString());
}
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
/// <summary>Test rename a snapshot to another existing snapshot</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRenameToExistingSnapshot()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2");
exception.Expect(typeof(SnapshotException));
string error = "The snapshot s2 already exists for directory " + sub1.ToString();
exception.ExpectMessage(error);
hdfs.RenameSnapshot(sub1, "s1", "s2");
}
/// <summary>
/// Adding as part of jira HDFS-5343
/// Test for checking the cat command on snapshot path it
/// cannot read a file beyond snapshot file length
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotFileLengthWithCatCommand()
{
FSDataInputStream fis = null;
FileStatus fileStatus = null;
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
hdfs.Mkdirs(sub);
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
DFSTestUtil.AppendFile(hdfs, file1, Blocksize);
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus
.GetLen());
fis = hdfs.Open(file1);
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead
);
fis.Close();
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead);
fis.Close();
TextWriter outBackup = System.Console.Out;
TextWriter errBackup = System.Console.Error;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bao));
Runtime.SetErr(new TextWriter(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
try
{
ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size
());
}
finally
{
Runtime.SetOut(outBackup);
Runtime.SetErr(errBackup);
}
}
public static void SetUp()
{
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
fs = cluster.GetFileSystem();
Path path1 = new Path(sub1, "dir1");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path1));
Path path2 = new Path(sub2, "dir2");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path2));
SnapshotTestHelper.CreateSnapshot(fs, sub1, "testSnapshot");
objInSnapshot = SnapshotTestHelper.GetSnapshotPath(sub1, "testSnapshot", "dir1");
}
/// <summary>
/// Test snapshot during file appending, before the corresponding
/// <see cref="Org.Apache.Hadoop.FS.FSDataOutputStream"/>
/// instance closes.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotWhileAppending()
{
Path file = new Path(dir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// 1. append without closing stream --> create snapshot
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot should be stored into s0's
// deleted list, with size BLOCKSIZE*2
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize());
INodeDirectory dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
DirectoryWithSnapshotFeature.DirectoryDiff last = dirNode.GetDiffs().GetLast();
// 2. append without closing stream
@out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
// re-check nodeInDeleted_S0
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 3. take snapshot --> close stream
hdfs.CreateSnapshot(dir, "s1");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot with size BLOCKSIZE*3 should
// have been stored in s1's deleted list
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
last = dirNode.GetDiffs().GetLast();
NUnit.Framework.Assert.IsTrue(fileNode.IsWithSnapshot());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 4. modify file --> append without closing stream --> take snapshot -->
// close stream
hdfs.SetReplication(file, (short)(Replication - 1));
@out = AppendFileWithoutClosing(file, Blocksize);
hdfs.CreateSnapshot(dir, "s2");
@out.Close();
// re-check the size of nodeInDeleted_S1
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
}
public virtual void TestSuccessiveSnapshotXAttrChanges()
{
// First snapshot
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
// Second snapshot
hdfs.SetXAttr(path, name1, newValue1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName2);
xattrs = hdfs.GetXAttrs(snapshotPath2);
NUnit.Framework.Assert.AreEqual(2, xattrs.Count);
Assert.AssertArrayEquals(newValue1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Third snapshot
hdfs.SetXAttr(path, name1, value1);
hdfs.RemoveXAttr(path, name2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName3);
xattrs = hdfs.GetXAttrs(snapshotPath3);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
// Check that the first and second snapshots'
// XAttrs have stayed constant
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
xattrs = hdfs.GetXAttrs(snapshotPath2);
NUnit.Framework.Assert.AreEqual(2, xattrs.Count);
Assert.AssertArrayEquals(newValue1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Remove the second snapshot and verify the first and
// third snapshots' XAttrs have stayed constant
hdfs.DeleteSnapshot(path, snapshotName2);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
xattrs = hdfs.GetXAttrs(snapshotPath3);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
hdfs.DeleteSnapshot(path, snapshotName);
hdfs.DeleteSnapshot(path, snapshotName3);
}
/// <summary>Test that users can copy a snapshot while preserving its xattrs.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestCopySnapshotShouldPreserveXAttrs()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
Path snapshotCopy = new Path(path.ToString() + "-copy");
string[] argv = new string[] { "-cp", "-px", snapshotPath.ToUri().ToString(), snapshotCopy
.ToUri().ToString() };
int ret = ToolRunner.Run(new FsShell(conf), argv);
NUnit.Framework.Assert.AreEqual("cp -px is not working on a snapshot", Success, ret
);
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotCopy);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
}
/// <summary>Test the snapshot limit of a single snapshottable directory.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotLimit()
{
int step = 1000;
string dirStr = "/testSnapshotLimit/dir";
Path dir = new Path(dirStr);
hdfs.Mkdirs(dir, new FsPermission((short)0x1ff));
hdfs.AllowSnapshot(dir);
int s = 0;
for (; s < DirectorySnapshottableFeature.SnapshotLimit; s++)
{
string snapshotName = "s" + s;
hdfs.CreateSnapshot(dir, snapshotName);
//create a file occasionally
if (s % step == 0)
{
Path file = new Path(dirStr, "f" + s);
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed);
}
}
try
{
hdfs.CreateSnapshot(dir, "s" + s);
NUnit.Framework.Assert.Fail("Expected to fail to create snapshot, but didn't.");
}
catch (IOException ioe)
{
SnapshotTestHelper.Log.Info("The exception is expected.", ioe);
}
for (int f = 0; f < DirectorySnapshottableFeature.SnapshotLimit; f += step)
{
string file = "f" + f;
s = Random.Next(step);
for (; s < DirectorySnapshottableFeature.SnapshotLimit; s += Random.Next(step))
{
Path p = SnapshotTestHelper.GetSnapshotPath(dir, "s" + s, file);
//the file #f exists in snapshot #s iff s > f.
NUnit.Framework.Assert.AreEqual(s > f, hdfs.Exists(p));
}
}
}
public virtual void TestDiffReportWithRenameAndAppend()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
DFSTestUtil.CreateFile(hdfs, foo, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
Path bar = new Path(root, "bar");
hdfs.Rename(foo, bar);
DFSTestUtil.AppendFile(hdfs, bar, 10);
// append 10 bytes
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
// we always put modification on the file before rename
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo"), DFSUtil.String2Bytes
("bar")));
}
public virtual void TestDiffReportWithRenameToNewDir()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
Path fileInFoo = new Path(foo, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
Path bar = new Path(root, "bar");
hdfs.Mkdirs(bar);
Path fileInBar = new Path(bar, "file");
hdfs.Rename(fileInFoo, fileInBar);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes("bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo/file"), DFSUtil.String2Bytes
("bar/file")));
}
/// <summary>
/// Rename snapshot(s), and check the correctness of the snapshot list within
/// <see cref="INodeDirectorySnapshottable"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotList()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create three snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s3");
// Rename s3 to s22
hdfs.RenameSnapshot(sub1, "s3", "s22");
// Check the snapshots list
INodeDirectory srcRoot = fsdir.GetINode(sub1.ToString()).AsDirectory();
CheckSnapshotList(srcRoot, new string[] { "s1", "s2", "s22" }, new string[] { "s1"
, "s2", "s22" });
// Rename s1 to s4
hdfs.RenameSnapshot(sub1, "s1", "s4");
CheckSnapshotList(srcRoot, new string[] { "s2", "s22", "s4" }, new string[] { "s4"
, "s2", "s22" });
// Rename s22 to s0
hdfs.RenameSnapshot(sub1, "s22", "s0");
CheckSnapshotList(srcRoot, new string[] { "s0", "s2", "s4" }, new string[] { "s4"
, "s2", "s0" });
}
public virtual void TestDisallowNestedSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
Path dir = new Path("/dir");
Path sub = new Path(dir, "sub");
hdfs.Mkdirs(sub);
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
Path file = new Path(sub, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
INode subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
hdfs.AllowSnapshot(sub);
subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.IsDirectory() && subNode.AsDirectory().IsSnapshottable
());
hdfs.DisallowSnapshot(sub);
subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
}
/// <exception cref="System.Exception"/>
public virtual void TestSetQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
Path sub = new Path(dir, "sub");
hdfs.Mkdirs(sub);
Path fileInSub = new Path(sub, "file");
DFSTestUtil.CreateFile(hdfs, fileInSub, Blocksize, Replication, seed);
INodeDirectory subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub
);
// subNode should be a INodeDirectory, but not an INodeDirectoryWithSnapshot
NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot());
hdfs.SetQuota(sub, long.MaxValue - 1, long.MaxValue - 1);
subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub);
NUnit.Framework.Assert.IsTrue(subNode.IsQuotaSet());
NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot());
}
/// <summary>Tests removing xattrs on a directory that has been snapshotted</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRemoveReadsCurrentState()
{
// Init
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
// Verify that current path reflects xattrs, snapshot doesn't
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
// Remove xattrs and verify one-by-one
hdfs.RemoveXAttr(path, name2);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 1);
Assert.AssertArrayEquals(value1, xattrs[name1]);
hdfs.RemoveXAttr(path, name1);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
}
