/// <summary>Test FileStatus of snapshot file before/after rename</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotRename()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshot for sub1
Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
Path ssPath = new Path(snapshotRoot, file1.GetName());
NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath));
FileStatus statusBeforeRename = hdfs.GetFileStatus(ssPath);
// Rename the snapshot
hdfs.RenameSnapshot(sub1, "s1", "s2");
// <sub1>/.snapshot/s1/file1 should no longer exist
NUnit.Framework.Assert.IsFalse(hdfs.Exists(ssPath));
snapshotRoot = SnapshotTestHelper.GetSnapshotRoot(sub1, "s2");
ssPath = new Path(snapshotRoot, file1.GetName());
// Instead, <sub1>/.snapshot/s2/file1 should exist
NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath));
FileStatus statusAfterRename = hdfs.GetFileStatus(ssPath);
// FileStatus of the snapshot should not change except the path
NUnit.Framework.Assert.IsFalse(statusBeforeRename.Equals(statusAfterRename));
statusBeforeRename.SetPath(statusAfterRename.GetPath());
NUnit.Framework.Assert.AreEqual(statusBeforeRename.ToString(), statusAfterRename.
ToString());
}
public virtual void TestXAttrForSnapshotRootAfterRemove()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
// Both original and snapshot have same XAttrs.
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Original XAttrs have been removed, but snapshot still has old XAttrs.
hdfs.RemoveXAttr(path, name1);
hdfs.RemoveXAttr(path, name2);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
Restart(false);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
Restart(true);
DoSnapshotRootRemovalAssertions(path, snapshotPath);
}
public virtual void TestDiffReportWithRenameAndSnapshotDeletion()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
// rename /foo to /foo2
Path foo2 = new Path(root, "foo2");
hdfs.Rename(foo, foo2);
// now /foo/bar becomes /foo2/bar
Path bar2 = new Path(foo2, "bar");
// delete snapshot s0 so that the withNameList inside of the WithCount node
// of foo becomes empty
hdfs.DeleteSnapshot(root, "s0");
// create snapshot s1 and rename bar again
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path bar3 = new Path(foo2, "bar-new");
hdfs.Rename(bar2, bar3);
// we always put modification on the file before rename
VerifyDiffReport(root, "s1", string.Empty, new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo2/bar"), DFSUtil.String2Bytes
("foo2/bar-new")));
}
public virtual void TestDeletionOfLaterBlocksWithZeroSizeFirstBlock()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
byte[] testData = Sharpen.Runtime.GetBytesForString("foo bar baz");
// Create a zero-length file.
DFSTestUtil.CreateFile(hdfs, bar, 0, Replication, 0L);
NUnit.Framework.Assert.AreEqual(0, fsdir.GetINode4Write(bar.ToString()).AsFile().
GetBlocks().Length);
// Create a snapshot that includes that file.
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s0");
// Extend that file.
FSDataOutputStream @out = hdfs.Append(bar);
@out.Write(testData);
@out.Close();
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(testData.Length, blks[0].GetNumBytes());
// Delete the file.
hdfs.Delete(bar, true);
// Now make sure that the NN can still save an fsimage successfully.
cluster.GetNameNode().GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter
, false);
cluster.GetNameNode().GetRpcServer().SaveNamespace();
}
/// <summary>Tests modifying xattrs on a directory that has been snapshotted</summary>
/// <exception cref="System.Exception"/>
public virtual void TestModifyReadsCurrentState()
{
// Init
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
// Verify that current path reflects xattrs, snapshot doesn't
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
// Modify each xattr and make sure it's reflected
hdfs.SetXAttr(path, name1, value2, EnumSet.Of(XAttrSetFlag.Replace));
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value2, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
hdfs.SetXAttr(path, name2, value1, EnumSet.Of(XAttrSetFlag.Replace));
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value2, xattrs[name1]);
Assert.AssertArrayEquals(value1, xattrs[name2]);
// Paranoia checks
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
hdfs.RemoveXAttr(path, name1);
hdfs.RemoveXAttr(path, name2);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
}
/// <summary>Test snapshot after file appending</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotAfterAppending()
{
Path file = new Path(dir, "file");
// 1. create snapshot --> create file --> append
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
// 2. create snapshot --> modify the file --> append
hdfs.CreateSnapshot(dir, "s1");
hdfs.SetReplication(file, (short)(Replication - 1));
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize());
// 3. create snapshot --> append
hdfs.CreateSnapshot(dir, "s2");
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize());
}
public virtual void TestDeletionWithZeroSizeBlock2()
{
Path foo = new Path("/foo");
Path subDir = new Path(foo, "sub");
Path bar = new Path(subDir, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, 0L);
hdfs.Append(bar);
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
ExtendedBlock previous = new ExtendedBlock(fsn.GetBlockPoolId(), blks[0]);
cluster.GetNameNodeRpc().AddBlock(bar.ToString(), hdfs.GetClient().GetClientName(
), previous, null, barNode.GetId(), null);
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s1");
barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(0, blks[1].GetNumBytes());
hdfs.Delete(subDir, true);
Path sbar = SnapshotTestHelper.GetSnapshotPath(foo, "s1", "sub/bar");
barNode = fsdir.GetINode(sbar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
}
/// <summary>Test replication number calculation for a file with snapshots.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestReplicationWithSnapshot()
{
short fileRep = 1;
// Create file1, set its replication to 1
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, fileRep, seed);
IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>();
// Change replication factor from 1 to 5. In the meanwhile, keep taking
// snapshots for sub1
for (; fileRep < Numdatanode;)
{
// Create snapshot for sub1
Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + fileRep);
Path snapshot = new Path(snapshotRoot, file1.GetName());
// Check the replication stored in the INode of the snapshot of file1
NUnit.Framework.Assert.AreEqual(fileRep, GetINodeFile(snapshot).GetFileReplication
());
snapshotRepMap[snapshot] = fileRep;
// Increase the replication factor by 1
hdfs.SetReplication(file1, ++fileRep);
// Check the replication for file1
CheckFileReplication(file1, fileRep, fileRep);
// Also check the replication for all the prior snapshots of file1
CheckSnapshotFileReplication(file1, snapshotRepMap, fileRep);
}
// Change replication factor back to 3.
hdfs.SetReplication(file1, Replication);
// Check the replication for file1
// Currently the max replication among snapshots should be 4
CheckFileReplication(file1, Replication, (short)(Numdatanode - 1));
// Also check the replication for all the prior snapshots of file1.
// Currently the max replication among snapshots should be 4
CheckSnapshotFileReplication(file1, snapshotRepMap, (short)(Numdatanode - 1));
}
/// <summary>
/// Test replication for a file with snapshots, also including the scenario
/// where the original file is deleted
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestReplicationAfterDeletion()
{
// Create file1, set its replication to 3
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>();
// Take 3 snapshots of sub1
for (int i = 1; i <= 3; i++)
{
Path root = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + i);
Path ssFile = new Path(root, file1.GetName());
snapshotRepMap[ssFile] = Replication;
}
// Check replication
CheckFileReplication(file1, Replication, Replication);
CheckSnapshotFileReplication(file1, snapshotRepMap, Replication);
// Delete file1
hdfs.Delete(file1, true);
// Check replication of snapshots
foreach (Path ss in snapshotRepMap.Keys)
{
INodeFile ssInode = GetINodeFile(ss);
// The replication number derived from the
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
NUnit.Framework.Assert.AreEqual(Replication, ssInode.GetBlockReplication());
// Also check the number derived from INodeFile#getFileReplication
NUnit.Framework.Assert.AreEqual(snapshotRepMap[ss], ssInode.GetFileReplication());
}
}
public virtual void TestDiffReportWithRename()
{
Path root = new Path("/");
Path sdir1 = new Path(root, "dir1");
Path sdir2 = new Path(root, "dir2");
Path foo = new Path(sdir1, "foo");
Path bar = new Path(foo, "bar");
hdfs.Mkdirs(bar);
hdfs.Mkdirs(sdir2);
// create snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
// /dir1/foo/bar -> /dir2/bar
Path bar2 = new Path(sdir2, "bar");
hdfs.Rename(bar, bar2);
// /dir1/foo -> /dir2/bar/foo
Path foo2 = new Path(bar2, "foo");
hdfs.Rename(foo, foo2);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
// let's delete /dir2 to make things more complicated
hdfs.Delete(sdir2, true);
VerifyDiffReport(root, "s1", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo"), DFSUtil.String2Bytes
("dir2/bar/foo")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes("dir1/foo/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo/bar"), DFSUtil
.String2Bytes("dir2/bar")));
}
public virtual void TestDiffReportWithRenameOutside()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
// create snapshot on /dir1
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s0");
// move bar into dir1
Path newBar = new Path(dir1, "newBar");
hdfs.Rename(bar, newBar);
// move foo out of dir1 into dir2
Path newFoo = new Path(dir2, "new");
hdfs.Rename(foo, newFoo);
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s1");
VerifyDiffReport(dir1, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes(newBar.GetName())), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes
(foo.GetName())));
}
public virtual void TestDiffReportWithRenameAndDelete()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
hdfs.Rename(fileInFoo, fileInBar, Options.Rename.Overwrite);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar/file")), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes
("dir1/foo/file"), DFSUtil.String2Bytes("dir2/bar/file")));
// delete bar
hdfs.Delete(bar, true);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
VerifyDiffReport(root, "s0", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir1/foo/file")));
}
public virtual void TestRenameWithIllegalName()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
string name1 = HdfsConstants.DotSnapshotDir;
try
{
hdfs.RenameSnapshot(sub1, "s1", name1);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given for rename"
);
}
catch (RemoteException e)
{
string errorMsg = "\"" + HdfsConstants.DotSnapshotDir + "\" is a reserved name.";
GenericTestUtils.AssertExceptionContains(errorMsg, e);
}
string errorMsg_1 = "Snapshot name cannot contain \"" + Path.Separator + "\"";
string[] badNames = new string[] { "foo" + Path.Separator, Path.Separator + "foo"
, Path.Separator, "foo" + Path.Separator + "bar" };
foreach (string badName in badNames)
{
try
{
hdfs.RenameSnapshot(sub1, "s1", badName);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given");
}
catch (RemoteException e)
{
GenericTestUtils.AssertExceptionContains(errorMsg_1, e);
}
}
}
public virtual void TestSetXAttrSnapshotPath()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
exception.Expect(typeof(SnapshotAccessControlException));
hdfs.SetXAttr(snapshotPath, name1, value1);
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
/// <summary>Test rename a snapshot to another existing snapshot</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRenameToExistingSnapshot()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2");
exception.Expect(typeof(SnapshotException));
string error = "The snapshot s2 already exists for directory " + sub1.ToString();
exception.ExpectMessage(error);
hdfs.RenameSnapshot(sub1, "s1", "s2");
}
public static void SetUp()
{
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
fs = cluster.GetFileSystem();
Path path1 = new Path(sub1, "dir1");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path1));
Path path2 = new Path(sub2, "dir2");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path2));
SnapshotTestHelper.CreateSnapshot(fs, sub1, "testSnapshot");
objInSnapshot = SnapshotTestHelper.GetSnapshotPath(sub1, "testSnapshot", "dir1");
}
/// <summary>
/// Test snapshot during file appending, before the corresponding
/// <see cref="Org.Apache.Hadoop.FS.FSDataOutputStream"/>
/// instance closes.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotWhileAppending()
{
Path file = new Path(dir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// 1. append without closing stream --> create snapshot
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot should be stored into s0's
// deleted list, with size BLOCKSIZE*2
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize());
INodeDirectory dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
DirectoryWithSnapshotFeature.DirectoryDiff last = dirNode.GetDiffs().GetLast();
// 2. append without closing stream
@out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
// re-check nodeInDeleted_S0
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 3. take snapshot --> close stream
hdfs.CreateSnapshot(dir, "s1");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot with size BLOCKSIZE*3 should
// have been stored in s1's deleted list
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
last = dirNode.GetDiffs().GetLast();
NUnit.Framework.Assert.IsTrue(fileNode.IsWithSnapshot());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 4. modify file --> append without closing stream --> take snapshot -->
// close stream
hdfs.SetReplication(file, (short)(Replication - 1));
@out = AppendFileWithoutClosing(file, Blocksize);
hdfs.CreateSnapshot(dir, "s2");
@out.Close();
// re-check the size of nodeInDeleted_S1
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
}
public virtual void TestSuccessiveSnapshotXAttrChanges()
{
// First snapshot
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
// Second snapshot
hdfs.SetXAttr(path, name1, newValue1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName2);
xattrs = hdfs.GetXAttrs(snapshotPath2);
NUnit.Framework.Assert.AreEqual(2, xattrs.Count);
Assert.AssertArrayEquals(newValue1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Third snapshot
hdfs.SetXAttr(path, name1, value1);
hdfs.RemoveXAttr(path, name2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName3);
xattrs = hdfs.GetXAttrs(snapshotPath3);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
// Check that the first and second snapshots'
// XAttrs have stayed constant
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
xattrs = hdfs.GetXAttrs(snapshotPath2);
NUnit.Framework.Assert.AreEqual(2, xattrs.Count);
Assert.AssertArrayEquals(newValue1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
// Remove the second snapshot and verify the first and
// third snapshots' XAttrs have stayed constant
hdfs.DeleteSnapshot(path, snapshotName2);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
xattrs = hdfs.GetXAttrs(snapshotPath3);
NUnit.Framework.Assert.AreEqual(1, xattrs.Count);
Assert.AssertArrayEquals(value1, xattrs[name1]);
hdfs.DeleteSnapshot(path, snapshotName);
hdfs.DeleteSnapshot(path, snapshotName3);
}
/// <summary>Test that users can copy a snapshot while preserving its xattrs.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestCopySnapshotShouldPreserveXAttrs()
{
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
Path snapshotCopy = new Path(path.ToString() + "-copy");
string[] argv = new string[] { "-cp", "-px", snapshotPath.ToUri().ToString(), snapshotCopy
.ToUri().ToString() };
int ret = ToolRunner.Run(new FsShell(conf), argv);
NUnit.Framework.Assert.AreEqual("cp -px is not working on a snapshot", Success, ret
);
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotCopy);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
}
public virtual void TestDiffReportWithRenameAndAppend()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
DFSTestUtil.CreateFile(hdfs, foo, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
Path bar = new Path(root, "bar");
hdfs.Rename(foo, bar);
DFSTestUtil.AppendFile(hdfs, bar, 10);
// append 10 bytes
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
// we always put modification on the file before rename
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo"), DFSUtil.String2Bytes
("bar")));
}
public virtual void TestDiffReportWithRenameToNewDir()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
Path fileInFoo = new Path(foo, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
Path bar = new Path(root, "bar");
hdfs.Mkdirs(bar);
Path fileInBar = new Path(bar, "file");
hdfs.Rename(fileInFoo, fileInBar);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes("bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo/file"), DFSUtil.String2Bytes
("bar/file")));
}
/// <exception cref="System.Exception"/>
public virtual void TestSetQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
Path sub = new Path(dir, "sub");
hdfs.Mkdirs(sub);
Path fileInSub = new Path(sub, "file");
DFSTestUtil.CreateFile(hdfs, fileInSub, Blocksize, Replication, seed);
INodeDirectory subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub
);
// subNode should be a INodeDirectory, but not an INodeDirectoryWithSnapshot
NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot());
hdfs.SetQuota(sub, long.MaxValue - 1, long.MaxValue - 1);
subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub);
NUnit.Framework.Assert.IsTrue(subNode.IsQuotaSet());
NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot());
}
/// <summary>
/// Rename snapshot(s), and check the correctness of the snapshot list within
/// <see cref="INodeDirectorySnapshottable"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotList()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create three snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2");
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s3");
// Rename s3 to s22
hdfs.RenameSnapshot(sub1, "s3", "s22");
// Check the snapshots list
INodeDirectory srcRoot = fsdir.GetINode(sub1.ToString()).AsDirectory();
CheckSnapshotList(srcRoot, new string[] { "s1", "s2", "s22" }, new string[] { "s1"
, "s2", "s22" });
// Rename s1 to s4
hdfs.RenameSnapshot(sub1, "s1", "s4");
CheckSnapshotList(srcRoot, new string[] { "s2", "s22", "s4" }, new string[] { "s4"
, "s2", "s22" });
// Rename s22 to s0
hdfs.RenameSnapshot(sub1, "s22", "s0");
CheckSnapshotList(srcRoot, new string[] { "s0", "s2", "s4" }, new string[] { "s4"
, "s2", "s0" });
}
public virtual void TestDisallowNestedSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
Path dir = new Path("/dir");
Path sub = new Path(dir, "sub");
hdfs.Mkdirs(sub);
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
Path file = new Path(sub, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
INode subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
hdfs.AllowSnapshot(sub);
subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.IsDirectory() && subNode.AsDirectory().IsSnapshottable
());
hdfs.DisallowSnapshot(sub);
subNode = fsdir.GetINode(sub.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
}
/// <summary>Tests removing xattrs on a directory that has been snapshotted</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRemoveReadsCurrentState()
{
// Init
FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0));
SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName);
hdfs.SetXAttr(path, name1, value1);
hdfs.SetXAttr(path, name2, value2);
// Verify that current path reflects xattrs, snapshot doesn't
IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
xattrs = hdfs.GetXAttrs(snapshotPath);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
// Remove xattrs and verify one-by-one
hdfs.RemoveXAttr(path, name2);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 1);
Assert.AssertArrayEquals(value1, xattrs[name1]);
hdfs.RemoveXAttr(path, name1);
xattrs = hdfs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
}
/// <summary>Test deleting a file with snapshots.</summary>
/// <remarks>
/// Test deleting a file with snapshots. Need to check the blocksMap to make
/// sure the corresponding record is updated correctly.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeletionWithSnapshots()
{
Path file0 = new Path(sub1, "file0");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(sub1, "sub2");
Path file2 = new Path(sub2, "file2");
Path file3 = new Path(sub1, "file3");
Path file4 = new Path(sub1, "file4");
Path file5 = new Path(sub1, "file5");
// Create file under sub1
DFSTestUtil.CreateFile(hdfs, file0, 4 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file1, 2 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, 3 * Blocksize, Replication, seed);
{
// Normal deletion
INodeFile f2 = AssertBlockCollection(file2.ToString(), 3, fsdir, blockmanager);
BlockInfoContiguous[] blocks = f2.GetBlocks();
hdfs.Delete(sub2, true);
// The INode should have been removed from the blocksMap
foreach (BlockInfoContiguous b in blocks)
{
NUnit.Framework.Assert.IsNull(blockmanager.GetBlockCollection(b));
}
}
// Create snapshots for sub1
string[] snapshots = new string[] { "s0", "s1", "s2" };
DFSTestUtil.CreateFile(hdfs, file3, 5 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[0]);
DFSTestUtil.CreateFile(hdfs, file4, 1 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[1]);
DFSTestUtil.CreateFile(hdfs, file5, 7 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[2]);
{
// set replication so that the inode should be replaced for snapshots
INodeFile f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.AreSame(typeof(INodeFile), f1.GetType());
hdfs.SetReplication(file1, (short)2);
f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.IsTrue(f1.IsWithSnapshot());
NUnit.Framework.Assert.IsFalse(f1.IsUnderConstruction());
}
// Check the block information for file0
INodeFile f0 = AssertBlockCollection(file0.ToString(), 4, fsdir, blockmanager);
BlockInfoContiguous[] blocks0 = f0.GetBlocks();
// Also check the block information for snapshot of file0
Path snapshotFile0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s0", file0.GetName
());
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Delete file0
hdfs.Delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
foreach (BlockInfoContiguous b_1 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_1));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Compare the INode in the blocksMap with INodes for snapshots
string s1f0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s1", file0.GetName()).ToString
();
AssertBlockCollection(s1f0, 4, fsdir, blockmanager);
// Delete snapshot s1
hdfs.DeleteSnapshot(sub1, "s1");
// Make sure the first block of file0 is still in blocksMap
foreach (BlockInfoContiguous b_2 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_2));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
try
{
INodeFile.ValueOf(fsdir.GetINode(s1f0), s1f0);
NUnit.Framework.Assert.Fail("Expect FileNotFoundException when identifying the INode in a deleted Snapshot"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("File does not exist: " + s1f0, e);
}
}
public virtual void TestGetBlockLocations()
{
Path root = new Path("/");
Path file = new Path("/file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// take a snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName
());
FileStatus status = hdfs.GetFileStatus(fileInSnapshot);
// make sure we record the size for the file
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// append data to file
DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1);
status = hdfs.GetFileStatus(fileInSnapshot);
// the size of snapshot file should still be BLOCKSIZE
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// the size of the file should be (2 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// call DFSClient#callGetBlockLocations for the file in snapshot
LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc
(), fileInSnapshot.ToString(), 0, long.MaxValue);
IList <LocatedBlock> blockList = blocks.GetLocatedBlocks();
// should be only one block
NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check the last block
LocatedBlock lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
// take another snapshot
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName
());
// append data to file without closing
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
status = hdfs.GetFileStatus(fileInSnapshot2);
// the size of snapshot file should be BLOCKSIZE*2-1
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// the size of the file should be (3 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), 0, long.MaxValue);
NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete());
blockList = blocks.GetLocatedBlocks();
// should be 2 blocks
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(2, blockList.Count);
// check the last block
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), Blocksize, 0);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check blocks for file being written
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString
(), 0, long.MaxValue);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(3, blockList.Count);
NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete());
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize());
@out.Close();
}
public virtual void TestUpdatePipelineAfterDelete()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path("/test-file");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
DFSOutputStream @out = null;
try
{
// Create a file and make sure a block is allocated for it.
@out = (DFSOutputStream)(fs.Create(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
// Create a snapshot that includes the file.
SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1");
// Grab the block info of this file for later use.
FSDataInputStream @in = null;
ExtendedBlock oldBlock = null;
try
{
@in = fs.Open(file);
oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// Allocate a new block ID/gen stamp so we can simulate pipeline
// recovery.
string clientName = ((DistributedFileSystem)fs).GetClient().GetClientName();
LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName
);
ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId
(), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp());
// Delete the file from the present FS. It will still exist the
// previously-created snapshot. This will log an OP_DELETE for the
// file in question.
fs.Delete(file, true);
// Simulate a pipeline recovery, wherein a new block is allocated
// for the existing block, resulting in an OP_UPDATE_BLOCKS being
// logged for the file in question.
try
{
namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations
(), newLocatedBlock.GetStorageIDs());
}
catch (IOException ioe)
{
// normal
GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction"
, ioe);
}
// Make sure the NN can restart with the edit logs as we have them now.
cluster.RestartNameNode(true);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}