public virtual void TestDeletionOfLaterBlocksWithZeroSizeFirstBlock() { Path foo = new Path("/foo"); Path bar = new Path(foo, "bar"); byte[] testData = Sharpen.Runtime.GetBytesForString("foo bar baz"); // Create a zero-length file. DFSTestUtil.CreateFile(hdfs, bar, 0, Replication, 0L); NUnit.Framework.Assert.AreEqual(0, fsdir.GetINode4Write(bar.ToString()).AsFile(). GetBlocks().Length); // Create a snapshot that includes that file. SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s0"); // Extend that file. FSDataOutputStream @out = hdfs.Append(bar); @out.Write(testData); @out.Close(); INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile(); BlockInfoContiguous[] blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(testData.Length, blks[0].GetNumBytes()); // Delete the file. hdfs.Delete(bar, true); // Now make sure that the NN can still save an fsimage successfully. cluster.GetNameNode().GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter , false); cluster.GetNameNode().GetRpcServer().SaveNamespace(); }
public virtual void TestSetXAttrSnapshotPath() { FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); exception.Expect(typeof(SnapshotAccessControlException)); hdfs.SetXAttr(snapshotPath, name1, value1); }
/// <summary> /// Test replication for a file with snapshots, also including the scenario /// where the original file is deleted /// </summary> /// <exception cref="System.Exception"/> public virtual void TestReplicationAfterDeletion() { // Create file1, set its replication to 3 DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>(); // Take 3 snapshots of sub1 for (int i = 1; i <= 3; i++) { Path root = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + i); Path ssFile = new Path(root, file1.GetName()); snapshotRepMap[ssFile] = Replication; } // Check replication CheckFileReplication(file1, Replication, Replication); CheckSnapshotFileReplication(file1, snapshotRepMap, Replication); // Delete file1 hdfs.Delete(file1, true); // Check replication of snapshots foreach (Path ss in snapshotRepMap.Keys) { INodeFile ssInode = GetINodeFile(ss); // The replication number derived from the // INodeFileWithLink#getBlockReplication should always == expectedBlockRep NUnit.Framework.Assert.AreEqual(Replication, ssInode.GetBlockReplication()); // Also check the number derived from INodeFile#getFileReplication NUnit.Framework.Assert.AreEqual(snapshotRepMap[ss], ssInode.GetFileReplication()); } }
public virtual void TestDeletionWithZeroSizeBlock2() { Path foo = new Path("/foo"); Path subDir = new Path(foo, "sub"); Path bar = new Path(subDir, "bar"); DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, 0L); hdfs.Append(bar); INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile(); BlockInfoContiguous[] blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); ExtendedBlock previous = new ExtendedBlock(fsn.GetBlockPoolId(), blks[0]); cluster.GetNameNodeRpc().AddBlock(bar.ToString(), hdfs.GetClient().GetClientName( ), previous, null, barNode.GetId(), null); SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s1"); barNode = fsdir.GetINode4Write(bar.ToString()).AsFile(); blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, blks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(0, blks[1].GetNumBytes()); hdfs.Delete(subDir, true); Path sbar = SnapshotTestHelper.GetSnapshotPath(foo, "s1", "sub/bar"); barNode = fsdir.GetINode(sbar.ToString()).AsFile(); blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes()); }
/// <summary>Tests modifying xattrs on a directory that has been snapshotted</summary> /// <exception cref="System.Exception"/> public virtual void TestModifyReadsCurrentState() { // Init FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); hdfs.SetXAttr(path, name1, value1); hdfs.SetXAttr(path, name2, value2); // Verify that current path reflects xattrs, snapshot doesn't IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); // Modify each xattr and make sure it's reflected hdfs.SetXAttr(path, name1, value2, EnumSet.Of(XAttrSetFlag.Replace)); xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value2, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); hdfs.SetXAttr(path, name2, value1, EnumSet.Of(XAttrSetFlag.Replace)); xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value2, xattrs[name1]); Assert.AssertArrayEquals(value1, xattrs[name2]); // Paranoia checks xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); hdfs.RemoveXAttr(path, name1); hdfs.RemoveXAttr(path, name2); xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); }
/// <summary>Test replication number calculation for a file with snapshots.</summary> /// <exception cref="System.Exception"/> public virtual void TestReplicationWithSnapshot() { short fileRep = 1; // Create file1, set its replication to 1 DFSTestUtil.CreateFile(hdfs, file1, Blocksize, fileRep, seed); IDictionary <Path, short> snapshotRepMap = new Dictionary <Path, short>(); // Change replication factor from 1 to 5. In the meanwhile, keep taking // snapshots for sub1 for (; fileRep < Numdatanode;) { // Create snapshot for sub1 Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s" + fileRep); Path snapshot = new Path(snapshotRoot, file1.GetName()); // Check the replication stored in the INode of the snapshot of file1 NUnit.Framework.Assert.AreEqual(fileRep, GetINodeFile(snapshot).GetFileReplication ()); snapshotRepMap[snapshot] = fileRep; // Increase the replication factor by 1 hdfs.SetReplication(file1, ++fileRep); // Check the replication for file1 CheckFileReplication(file1, fileRep, fileRep); // Also check the replication for all the prior snapshots of file1 CheckSnapshotFileReplication(file1, snapshotRepMap, fileRep); } // Change replication factor back to 3. hdfs.SetReplication(file1, Replication); // Check the replication for file1 // Currently the max replication among snapshots should be 4 CheckFileReplication(file1, Replication, (short)(Numdatanode - 1)); // Also check the replication for all the prior snapshots of file1. // Currently the max replication among snapshots should be 4 CheckSnapshotFileReplication(file1, snapshotRepMap, (short)(Numdatanode - 1)); }
/// <summary>Test snapshot after file appending</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotAfterAppending() { Path file = new Path(dir, "file"); // 1. create snapshot --> create file --> append SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); DFSTestUtil.AppendFile(hdfs, file, Blocksize); INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString()); // 2. create snapshot --> modify the file --> append hdfs.CreateSnapshot(dir, "s1"); hdfs.SetReplication(file, (short)(Replication - 1)); DFSTestUtil.AppendFile(hdfs, file, Blocksize); // check corresponding inodes fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication()); NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize()); // 3. create snapshot --> append hdfs.CreateSnapshot(dir, "s2"); DFSTestUtil.AppendFile(hdfs, file, Blocksize); // check corresponding inodes fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication()); NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize()); }
public TestINodeFileUnderConstructionWithSnapshot() { { ((Log4JLogger)INode.Log).GetLogger().SetLevel(Level.All); SnapshotTestHelper.DisableLogs(); } }
public virtual void TestDiffReportWithRenameOutside() { Path root = new Path("/"); Path dir1 = new Path(root, "dir1"); Path dir2 = new Path(root, "dir2"); Path foo = new Path(dir1, "foo"); Path fileInFoo = new Path(foo, "file"); Path bar = new Path(dir2, "bar"); Path fileInBar = new Path(bar, "file"); DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed); DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed); // create snapshot on /dir1 SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s0"); // move bar into dir1 Path newBar = new Path(dir1, "newBar"); hdfs.Rename(bar, newBar); // move foo out of dir1 into dir2 Path newFoo = new Path(dir2, "new"); hdfs.Rename(foo, newFoo); SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s1"); VerifyDiffReport(dir1, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes(newBar.GetName())), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes (foo.GetName()))); }
public virtual void TestDiffReportWithRename() { Path root = new Path("/"); Path sdir1 = new Path(root, "dir1"); Path sdir2 = new Path(root, "dir2"); Path foo = new Path(sdir1, "foo"); Path bar = new Path(foo, "bar"); hdfs.Mkdirs(bar); hdfs.Mkdirs(sdir2); // create snapshot on root SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); // /dir1/foo/bar -> /dir2/bar Path bar2 = new Path(sdir2, "bar"); hdfs.Rename(bar, bar2); // /dir1/foo -> /dir2/bar/foo Path foo2 = new Path(bar2, "foo"); hdfs.Rename(foo, foo2); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2"); // let's delete /dir2 to make things more complicated hdfs.Delete(sdir2, true); VerifyDiffReport(root, "s1", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo"), DFSUtil.String2Bytes ("dir2/bar/foo")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes("dir1/foo/bar")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("dir1/foo/bar"), DFSUtil .String2Bytes("dir2/bar"))); }
public virtual void TestRenameWithIllegalName() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create snapshots for sub1 SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); string name1 = HdfsConstants.DotSnapshotDir; try { hdfs.RenameSnapshot(sub1, "s1", name1); NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given for rename" ); } catch (RemoteException e) { string errorMsg = "\"" + HdfsConstants.DotSnapshotDir + "\" is a reserved name."; GenericTestUtils.AssertExceptionContains(errorMsg, e); } string errorMsg_1 = "Snapshot name cannot contain \"" + Path.Separator + "\""; string[] badNames = new string[] { "foo" + Path.Separator, Path.Separator + "foo" , Path.Separator, "foo" + Path.Separator + "bar" }; foreach (string badName in badNames) { try { hdfs.RenameSnapshot(sub1, "s1", badName); NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given"); } catch (RemoteException e) { GenericTestUtils.AssertExceptionContains(errorMsg_1, e); } } }
public virtual void TestDiffReportWithRenameAndDelete() { Path root = new Path("/"); Path dir1 = new Path(root, "dir1"); Path dir2 = new Path(root, "dir2"); Path foo = new Path(dir1, "foo"); Path fileInFoo = new Path(foo, "file"); Path bar = new Path(dir2, "bar"); Path fileInBar = new Path(bar, "file"); DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed); DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0"); hdfs.Rename(fileInFoo, fileInBar, Options.Rename.Overwrite); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar/file")), new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes ("dir1/foo/file"), DFSUtil.String2Bytes("dir2/bar/file"))); // delete bar hdfs.Delete(bar, true); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2"); VerifyDiffReport(root, "s0", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir1/foo/file"))); }
/// <summary>Test FileStatus of snapshot file before/after rename</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotRename() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create snapshot for sub1 Path snapshotRoot = SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); Path ssPath = new Path(snapshotRoot, file1.GetName()); NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath)); FileStatus statusBeforeRename = hdfs.GetFileStatus(ssPath); // Rename the snapshot hdfs.RenameSnapshot(sub1, "s1", "s2"); // <sub1>/.snapshot/s1/file1 should no longer exist NUnit.Framework.Assert.IsFalse(hdfs.Exists(ssPath)); snapshotRoot = SnapshotTestHelper.GetSnapshotRoot(sub1, "s2"); ssPath = new Path(snapshotRoot, file1.GetName()); // Instead, <sub1>/.snapshot/s2/file1 should exist NUnit.Framework.Assert.IsTrue(hdfs.Exists(ssPath)); FileStatus statusAfterRename = hdfs.GetFileStatus(ssPath); // FileStatus of the snapshot should not change except the path NUnit.Framework.Assert.IsFalse(statusBeforeRename.Equals(statusAfterRename)); statusBeforeRename.SetPath(statusAfterRename.GetPath()); NUnit.Framework.Assert.AreEqual(statusBeforeRename.ToString(), statusAfterRename. ToString()); }
public virtual void TestXAttrForSnapshotRootAfterRemove() { FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); hdfs.SetXAttr(path, name1, value1); hdfs.SetXAttr(path, name2, value2); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); // Both original and snapshot have same XAttrs. IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); // Original XAttrs have been removed, but snapshot still has old XAttrs. hdfs.RemoveXAttr(path, name1); hdfs.RemoveXAttr(path, name2); DoSnapshotRootRemovalAssertions(path, snapshotPath); Restart(false); DoSnapshotRootRemovalAssertions(path, snapshotPath); Restart(true); DoSnapshotRootRemovalAssertions(path, snapshotPath); }
public virtual void TestDiffReportWithRenameAndSnapshotDeletion() { Path root = new Path("/"); Path foo = new Path(root, "foo"); Path bar = new Path(foo, "bar"); DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, seed); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0"); // rename /foo to /foo2 Path foo2 = new Path(root, "foo2"); hdfs.Rename(foo, foo2); // now /foo/bar becomes /foo2/bar Path bar2 = new Path(foo2, "bar"); // delete snapshot s0 so that the withNameList inside of the WithCount node // of foo becomes empty hdfs.DeleteSnapshot(root, "s0"); // create snapshot s1 and rename bar again SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); Path bar3 = new Path(foo2, "bar-new"); hdfs.Rename(bar2, bar3); // we always put modification on the file before rename VerifyDiffReport(root, "s1", string.Empty, new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo2")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo2/bar"), DFSUtil.String2Bytes ("foo2/bar-new"))); }
/// <exception cref="System.Exception"/> internal virtual void AssertINodeNullInSnapshots(Path path, params string[] snapshots ) { foreach (string s in snapshots) { AssertINodeNull(SnapshotTestHelper.GetSnapshotPath(path.GetParent(), s, path.GetName ()).ToString()); } }
public virtual void TestClearQuota() { Path dir = new Path("/TestSnapshot"); hdfs.Mkdirs(dir); hdfs.AllowSnapshot(dir); hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet); INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable()); NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count); hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1 ); dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable()); NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count); hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset); dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable()); NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count); // allow snapshot on dir and create snapshot s1 SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1"); // clear quota of dir hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset); // dir should still be a snapshottable directory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable()); NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count); SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing(); NUnit.Framework.Assert.AreEqual(1, status.Length); NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath()); Path subDir = new Path(dir, "sub"); hdfs.Mkdirs(subDir); hdfs.CreateSnapshot(dir, "s2"); Path file = new Path(subDir, "file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset); INode subNode = fsdir.GetINode4Write(subDir.ToString()); NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot()); IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory( ).GetDiffs().AsList(); NUnit.Framework.Assert.AreEqual(1, diffList.Count); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot (DFSUtil.String2Bytes("s2")); NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId()); IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created ); NUnit.Framework.Assert.AreEqual(1, createdList.Count); NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList [0]); }
/// <summary>Test rename a snapshot to another existing snapshot</summary> /// <exception cref="System.Exception"/> public virtual void TestRenameToExistingSnapshot() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create snapshots for sub1 SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2"); exception.Expect(typeof(SnapshotException)); string error = "The snapshot s2 already exists for directory " + sub1.ToString(); exception.ExpectMessage(error); hdfs.RenameSnapshot(sub1, "s1", "s2"); }
/// <summary> /// Adding as part of jira HDFS-5343 /// Test for checking the cat command on snapshot path it /// cannot read a file beyond snapshot file length /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotFileLengthWithCatCommand() { FSDataInputStream fis = null; FileStatus fileStatus = null; int bytesRead; byte[] buffer = new byte[Blocksize * 8]; hdfs.Mkdirs(sub); Path file1 = new Path(sub, file1Name); DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed); hdfs.AllowSnapshot(sub); hdfs.CreateSnapshot(sub, snapshot1); DFSTestUtil.AppendFile(hdfs, file1, Blocksize); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.GetFileStatus(file1); NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus .GetLen()); fis = hdfs.Open(file1); bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead ); fis.Close(); Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name); fis = hdfs.Open(file1snap1); fileStatus = hdfs.GetFileStatus(file1snap1); NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize); // Make sure we can only read up to the snapshot length. bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead); fis.Close(); TextWriter outBackup = System.Console.Out; TextWriter errBackup = System.Console.Error; ByteArrayOutputStream bao = new ByteArrayOutputStream(); Runtime.SetOut(new TextWriter(bao)); Runtime.SetErr(new TextWriter(bao)); // Make sure we can cat the file upto to snapshot length FsShell shell = new FsShell(); try { ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" }); NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size ()); } finally { Runtime.SetOut(outBackup); Runtime.SetErr(errBackup); } }
public static void SetUp() { conf = new Configuration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); fs = cluster.GetFileSystem(); Path path1 = new Path(sub1, "dir1"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path1)); Path path2 = new Path(sub2, "dir2"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path2)); SnapshotTestHelper.CreateSnapshot(fs, sub1, "testSnapshot"); objInSnapshot = SnapshotTestHelper.GetSnapshotPath(sub1, "testSnapshot", "dir1"); }
/// <summary> /// Test snapshot during file appending, before the corresponding /// <see cref="Org.Apache.Hadoop.FS.FSDataOutputStream"/> /// instance closes. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotWhileAppending() { Path file = new Path(dir, "file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); // 1. append without closing stream --> create snapshot HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0"); @out.Close(); // check: an INodeFileUnderConstructionWithSnapshot should be stored into s0's // deleted list, with size BLOCKSIZE*2 INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize()); INodeDirectory dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); DirectoryWithSnapshotFeature.DirectoryDiff last = dirNode.GetDiffs().GetLast(); // 2. append without closing stream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); // re-check nodeInDeleted_S0 dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize(last.GetSnapshotId ())); // 3. take snapshot --> close stream hdfs.CreateSnapshot(dir, "s1"); @out.Close(); // check: an INodeFileUnderConstructionWithSnapshot with size BLOCKSIZE*3 should // have been stored in s1's deleted list fileNode = (INodeFile)fsdir.GetINode(file.ToString()); dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); last = dirNode.GetDiffs().GetLast(); NUnit.Framework.Assert.IsTrue(fileNode.IsWithSnapshot()); NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId ())); // 4. modify file --> append without closing stream --> take snapshot --> // close stream hdfs.SetReplication(file, (short)(Replication - 1)); @out = AppendFileWithoutClosing(file, Blocksize); hdfs.CreateSnapshot(dir, "s2"); @out.Close(); // re-check the size of nodeInDeleted_S1 NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId ())); }
public virtual void TestSuccessiveSnapshotXAttrChanges() { // First snapshot FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); hdfs.SetXAttr(path, name1, value1); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(1, xattrs.Count); Assert.AssertArrayEquals(value1, xattrs[name1]); // Second snapshot hdfs.SetXAttr(path, name1, newValue1); hdfs.SetXAttr(path, name2, value2); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName2); xattrs = hdfs.GetXAttrs(snapshotPath2); NUnit.Framework.Assert.AreEqual(2, xattrs.Count); Assert.AssertArrayEquals(newValue1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); // Third snapshot hdfs.SetXAttr(path, name1, value1); hdfs.RemoveXAttr(path, name2); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName3); xattrs = hdfs.GetXAttrs(snapshotPath3); NUnit.Framework.Assert.AreEqual(1, xattrs.Count); Assert.AssertArrayEquals(value1, xattrs[name1]); // Check that the first and second snapshots' // XAttrs have stayed constant xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(1, xattrs.Count); Assert.AssertArrayEquals(value1, xattrs[name1]); xattrs = hdfs.GetXAttrs(snapshotPath2); NUnit.Framework.Assert.AreEqual(2, xattrs.Count); Assert.AssertArrayEquals(newValue1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); // Remove the second snapshot and verify the first and // third snapshots' XAttrs have stayed constant hdfs.DeleteSnapshot(path, snapshotName2); xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(1, xattrs.Count); Assert.AssertArrayEquals(value1, xattrs[name1]); xattrs = hdfs.GetXAttrs(snapshotPath3); NUnit.Framework.Assert.AreEqual(1, xattrs.Count); Assert.AssertArrayEquals(value1, xattrs[name1]); hdfs.DeleteSnapshot(path, snapshotName); hdfs.DeleteSnapshot(path, snapshotName3); }
/// <summary>Test that users can copy a snapshot while preserving its xattrs.</summary> /// <exception cref="System.Exception"/> public virtual void TestCopySnapshotShouldPreserveXAttrs() { FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); hdfs.SetXAttr(path, name1, value1); hdfs.SetXAttr(path, name2, value2); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); Path snapshotCopy = new Path(path.ToString() + "-copy"); string[] argv = new string[] { "-cp", "-px", snapshotPath.ToUri().ToString(), snapshotCopy .ToUri().ToString() }; int ret = ToolRunner.Run(new FsShell(conf), argv); NUnit.Framework.Assert.AreEqual("cp -px is not working on a snapshot", Success, ret ); IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(snapshotCopy); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); }
/// <summary>Test the snapshot limit of a single snapshottable directory.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotLimit() { int step = 1000; string dirStr = "/testSnapshotLimit/dir"; Path dir = new Path(dirStr); hdfs.Mkdirs(dir, new FsPermission((short)0x1ff)); hdfs.AllowSnapshot(dir); int s = 0; for (; s < DirectorySnapshottableFeature.SnapshotLimit; s++) { string snapshotName = "s" + s; hdfs.CreateSnapshot(dir, snapshotName); //create a file occasionally if (s % step == 0) { Path file = new Path(dirStr, "f" + s); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed); } } try { hdfs.CreateSnapshot(dir, "s" + s); NUnit.Framework.Assert.Fail("Expected to fail to create snapshot, but didn't."); } catch (IOException ioe) { SnapshotTestHelper.Log.Info("The exception is expected.", ioe); } for (int f = 0; f < DirectorySnapshottableFeature.SnapshotLimit; f += step) { string file = "f" + f; s = Random.Next(step); for (; s < DirectorySnapshottableFeature.SnapshotLimit; s += Random.Next(step)) { Path p = SnapshotTestHelper.GetSnapshotPath(dir, "s" + s, file); //the file #f exists in snapshot #s iff s > f. NUnit.Framework.Assert.AreEqual(s > f, hdfs.Exists(p)); } } }
public virtual void TestDiffReportWithRenameAndAppend() { Path root = new Path("/"); Path foo = new Path(root, "foo"); DFSTestUtil.CreateFile(hdfs, foo, Blocksize, Replication, seed); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0"); Path bar = new Path(root, "bar"); hdfs.Rename(foo, bar); DFSTestUtil.AppendFile(hdfs, bar, 10); // append 10 bytes SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); // we always put modification on the file before rename VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo"), DFSUtil.String2Bytes ("bar"))); }
public virtual void TestDiffReportWithRenameToNewDir() { Path root = new Path("/"); Path foo = new Path(root, "foo"); Path fileInFoo = new Path(foo, "file"); DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0"); Path bar = new Path(root, "bar"); hdfs.Mkdirs(bar); Path fileInBar = new Path(bar, "file"); hdfs.Rename(fileInFoo, fileInBar); SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType .Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes("bar")), new SnapshotDiffReport.DiffReportEntry (SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo/file"), DFSUtil.String2Bytes ("bar/file"))); }
/// <summary> /// Rename snapshot(s), and check the correctness of the snapshot list within /// <see cref="INodeDirectorySnapshottable"/> /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotList() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create three snapshots for sub1 SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2"); SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s3"); // Rename s3 to s22 hdfs.RenameSnapshot(sub1, "s3", "s22"); // Check the snapshots list INodeDirectory srcRoot = fsdir.GetINode(sub1.ToString()).AsDirectory(); CheckSnapshotList(srcRoot, new string[] { "s1", "s2", "s22" }, new string[] { "s1" , "s2", "s22" }); // Rename s1 to s4 hdfs.RenameSnapshot(sub1, "s1", "s4"); CheckSnapshotList(srcRoot, new string[] { "s2", "s22", "s4" }, new string[] { "s4" , "s2", "s22" }); // Rename s22 to s0 hdfs.RenameSnapshot(sub1, "s22", "s0"); CheckSnapshotList(srcRoot, new string[] { "s0", "s2", "s4" }, new string[] { "s4" , "s2", "s0" }); }
public virtual void TestDisallowNestedSnapshottableDir() { cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true); Path dir = new Path("/dir"); Path sub = new Path(dir, "sub"); hdfs.Mkdirs(sub); SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1"); Path file = new Path(sub, "file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed); FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory(); INode subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot()); hdfs.AllowSnapshot(sub); subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.IsDirectory() && subNode.AsDirectory().IsSnapshottable ()); hdfs.DisallowSnapshot(sub); subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot()); }
/// <exception cref="System.Exception"/> public virtual void TestSetQuota() { Path dir = new Path("/TestSnapshot"); hdfs.Mkdirs(dir); // allow snapshot on dir and create snapshot s1 SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1"); Path sub = new Path(dir, "sub"); hdfs.Mkdirs(sub); Path fileInSub = new Path(sub, "file"); DFSTestUtil.CreateFile(hdfs, fileInSub, Blocksize, Replication, seed); INodeDirectory subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub ); // subNode should be a INodeDirectory, but not an INodeDirectoryWithSnapshot NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot()); hdfs.SetQuota(sub, long.MaxValue - 1, long.MaxValue - 1); subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub); NUnit.Framework.Assert.IsTrue(subNode.IsQuotaSet()); NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot()); }
/// <summary>Tests removing xattrs on a directory that has been snapshotted</summary> /// <exception cref="System.Exception"/> public virtual void TestRemoveReadsCurrentState() { // Init FileSystem.Mkdirs(hdfs, path, FsPermission.CreateImmutable((short)0x1c0)); SnapshotTestHelper.CreateSnapshot(hdfs, path, snapshotName); hdfs.SetXAttr(path, name1, value1); hdfs.SetXAttr(path, name2, value2); // Verify that current path reflects xattrs, snapshot doesn't IDictionary <string, byte[]> xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 2); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); xattrs = hdfs.GetXAttrs(snapshotPath); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); // Remove xattrs and verify one-by-one hdfs.RemoveXAttr(path, name2); xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 1); Assert.AssertArrayEquals(value1, xattrs[name1]); hdfs.RemoveXAttr(path, name1); xattrs = hdfs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); }