/// <summary>Test snapshot after file appending</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotAfterAppending() { Path file = new Path(dir, "file"); // 1. create snapshot --> create file --> append SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); DFSTestUtil.AppendFile(hdfs, file, Blocksize); INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString()); // 2. create snapshot --> modify the file --> append hdfs.CreateSnapshot(dir, "s1"); hdfs.SetReplication(file, (short)(Replication - 1)); DFSTestUtil.AppendFile(hdfs, file, Blocksize); // check corresponding inodes fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication()); NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize()); // 3. create snapshot --> append hdfs.CreateSnapshot(dir, "s2"); DFSTestUtil.AppendFile(hdfs, file, Blocksize); // check corresponding inodes fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication()); NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize()); }
/// <summary>Get a listing of all the snapshots of a snapshottable directory</summary> /// <exception cref="System.IO.IOException"/> private static DirectoryListing GetSnapshotsListing(FSDirectory fsd, string src, byte[] startAfter) { Preconditions.CheckState(fsd.HasReadLock()); Preconditions.CheckArgument(src.EndsWith(HdfsConstants.SeparatorDotSnapshotDir), "%s does not end with %s", src, HdfsConstants.SeparatorDotSnapshotDir); string dirPath = FSDirectory.NormalizePath(Sharpen.Runtime.Substring(src, 0, src. Length - HdfsConstants.DotSnapshotDir.Length)); INode node = fsd.GetINode(dirPath); INodeDirectory dirNode = INodeDirectory.ValueOf(node, dirPath); DirectorySnapshottableFeature sf = dirNode.GetDirectorySnapshottableFeature(); if (sf == null) { throw new SnapshotException("Directory is not a snapshottable directory: " + dirPath ); } ReadOnlyList <Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot> snapshots = sf.GetSnapshotList(); int skipSize = ReadOnlyList.Util.BinarySearch(snapshots, startAfter); skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1; int numOfListing = Math.Min(snapshots.Size() - skipSize, fsd.GetLsLimit()); HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing; i++) { Snapshot.Root sRoot = snapshots.Get(i + skipSize).GetRoot(); listing[i] = CreateFileStatus(fsd, src, sRoot.GetLocalNameBytes(), sRoot, BlockStoragePolicySuite .IdUnspecified, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId , false, INodesInPath.FromINode(sRoot)); } return(new DirectoryListing(listing, snapshots.Size() - skipSize - numOfListing)); }
/// <summary>Check the replication of a given file.</summary> /// <remarks> /// Check the replication of a given file. We test both /// <see cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeFile.GetFileReplication()" /// /> /// and /// <see cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeFile.GetBlockReplication() /// "/> /// . /// </remarks> /// <param name="file">The given file</param> /// <param name="replication">The expected replication number</param> /// <param name="blockReplication">The expected replication number for the block</param> /// <exception cref="System.Exception"/> private void CheckFileReplication(Path file, short replication, short blockReplication ) { // Get FileStatus of file1, and identify the replication number of file1. // Note that the replication number in FileStatus was derived from // INodeFile#getFileReplication(). short fileReplication = hdfs.GetFileStatus(file1).GetReplication(); NUnit.Framework.Assert.AreEqual(replication, fileReplication); // Check the correctness of getBlockReplication() INode inode = fsdir.GetINode(file1.ToString()); NUnit.Framework.Assert.IsTrue(inode is INodeFile); NUnit.Framework.Assert.AreEqual(blockReplication, ((INodeFile)inode).GetBlockReplication ()); }
/// <exception cref="System.IO.IOException"/> public static void DumpTree2File(FSDirectory fsdir, FilePath f) { PrintWriter @out = new PrintWriter(new FileWriter(f, false), true); fsdir.GetINode("/").DumpTreeRecursively(@out, new StringBuilder(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId); @out.Close(); }
/// <summary>Test allow-snapshot operation.</summary> /// <exception cref="System.Exception"/> public virtual void TestAllowSnapshot() { string pathStr = sub1.ToString(); INode before = fsdir.GetINode(pathStr); // Before a directory is snapshottable NUnit.Framework.Assert.IsFalse(before.AsDirectory().IsSnapshottable()); // After a directory is snapshottable Path path = new Path(pathStr); hdfs.AllowSnapshot(path); { INode after = fsdir.GetINode(pathStr); NUnit.Framework.Assert.IsTrue(after.AsDirectory().IsSnapshottable()); } hdfs.DisallowSnapshot(path); { INode after = fsdir.GetINode(pathStr); NUnit.Framework.Assert.IsFalse(after.AsDirectory().IsSnapshottable()); } }
/// <exception cref="System.Exception"/> internal static INodeFile AssertBlockCollection(string path, int numBlocks, FSDirectory dir, BlockManager blkManager) { INodeFile file = INodeFile.ValueOf(dir.GetINode(path), path); NUnit.Framework.Assert.AreEqual(numBlocks, file.GetBlocks().Length); foreach (BlockInfoContiguous b in file.GetBlocks()) { AssertBlockCollection(blkManager, file, b); } return(file); }
/// <summary>Test append over storage quota does not mark file as UC or create lease</summary> /// <exception cref="System.Exception"/> public virtual void TestAppendOverStorageQuota() { Path dir = new Path("/TestAppendOverQuota"); Path file = new Path(dir, "file"); // create partial block file dfs.Mkdirs(dir); DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed); // lower quota to cause exception when appending to partial block dfs.SetQuota(dir, long.MaxValue - 1, 1); INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory(); long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace (); try { DFSTestUtil.AppendFile(dfs, file, Blocksize); NUnit.Framework.Assert.Fail("append didn't fail"); } catch (DSQuotaExceededException) { } // ignore // check that the file exists, isn't UC, and has no dangling lease INodeFile inode = fsdir.GetINode(file.ToString()).AsFile(); NUnit.Framework.Assert.IsNotNull(inode); NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction()); NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem(). GetLeaseManager().GetLeaseByPath(file.ToString())); // make sure the quota usage is unchanged long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace (); NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed); // make sure edits aren't corrupted dfs.RecoverLease(file); cluster.RestartNameNodes(); }
public virtual void TestDisallowNestedSnapshottableDir() { cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true); Path dir = new Path("/dir"); Path sub = new Path(dir, "sub"); hdfs.Mkdirs(sub); SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1"); Path file = new Path(sub, "file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed); FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory(); INode subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot()); hdfs.AllowSnapshot(sub); subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.IsDirectory() && subNode.AsDirectory().IsSnapshottable ()); hdfs.DisallowSnapshot(sub); subNode = fsdir.GetINode(sub.ToString()); NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot()); }
/// <exception cref="System.Exception"/> public virtual void TestSetQuota() { Path dir = new Path("/TestSnapshot"); hdfs.Mkdirs(dir); // allow snapshot on dir and create snapshot s1 SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1"); Path sub = new Path(dir, "sub"); hdfs.Mkdirs(sub); Path fileInSub = new Path(sub, "file"); DFSTestUtil.CreateFile(hdfs, fileInSub, Blocksize, Replication, seed); INodeDirectory subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub ); // subNode should be a INodeDirectory, but not an INodeDirectoryWithSnapshot NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot()); hdfs.SetQuota(sub, long.MaxValue - 1, long.MaxValue - 1); subNode = INodeDirectory.ValueOf(fsdir.GetINode(sub.ToString()), sub); NUnit.Framework.Assert.IsTrue(subNode.IsQuotaSet()); NUnit.Framework.Assert.IsFalse(subNode.IsWithSnapshot()); }
/// <summary> /// Rename snapshot(s), and check the correctness of the snapshot list within /// <see cref="INodeDirectorySnapshottable"/> /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotList() { DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed); // Create three snapshots for sub1 SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1"); SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s2"); SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s3"); // Rename s3 to s22 hdfs.RenameSnapshot(sub1, "s3", "s22"); // Check the snapshots list INodeDirectory srcRoot = fsdir.GetINode(sub1.ToString()).AsDirectory(); CheckSnapshotList(srcRoot, new string[] { "s1", "s2", "s22" }, new string[] { "s1" , "s2", "s22" }); // Rename s1 to s4 hdfs.RenameSnapshot(sub1, "s1", "s4"); CheckSnapshotList(srcRoot, new string[] { "s2", "s22", "s4" }, new string[] { "s4" , "s2", "s22" }); // Rename s22 to s0 hdfs.RenameSnapshot(sub1, "s22", "s0"); CheckSnapshotList(srcRoot, new string[] { "s0", "s2", "s4" }, new string[] { "s4" , "s2", "s0" }); }
public virtual void TestDumpTree() { INode root = fsdir.GetINode("/"); Log.Info("Original tree"); StringBuilder b1 = root.DumpTreeRecursively(); System.Console.Out.WriteLine("b1=" + b1); BufferedReader @in = new BufferedReader(new StringReader(b1.ToString())); string line = @in.ReadLine(); CheckClassName(line); for (; (line = @in.ReadLine()) != null;) { line = line.Trim(); if (!line.IsEmpty() && !line.Contains("snapshot")) { NUnit.Framework.Assert.IsTrue("line=" + line, line.StartsWith(INodeDirectory.DumptreeLastItem ) || line.StartsWith(INodeDirectory.DumptreeExceptLastItem)); CheckClassName(line); } } }
/// <exception cref="System.Exception"/> internal virtual void AssertINodeNull(string path) { NUnit.Framework.Assert.IsNull(fsdir.GetINode(path)); }
/// <summary>Scan all CacheDirectives.</summary> /// <remarks> /// Scan all CacheDirectives. Use the information to figure out /// what cache replication factor each block should have. /// </remarks> private void RescanCacheDirectives() { FSDirectory fsDir = namesystem.GetFSDirectory(); long now = new DateTime().GetTime(); foreach (CacheDirective directive in cacheManager.GetCacheDirectives()) { scannedDirectives++; // Skip processing this entry if it has expired if (directive.GetExpiryTime() > 0 && directive.GetExpiryTime() <= now) { Log.Debug("Directive {}: the directive expired at {} (now = {})", directive.GetId (), directive.GetExpiryTime(), now); continue; } string path = directive.GetPath(); INode node; try { node = fsDir.GetINode(path); } catch (UnresolvedLinkException) { // We don't cache through symlinks Log.Debug("Directive {}: got UnresolvedLinkException while resolving " + "path {}" , directive.GetId(), path); continue; } if (node == null) { Log.Debug("Directive {}: No inode found at {}", directive.GetId(), path); } else { if (node.IsDirectory()) { INodeDirectory dir = node.AsDirectory(); ReadOnlyList <INode> children = dir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId); foreach (INode child in children) { if (child.IsFile()) { RescanFile(directive, child.AsFile()); } } } else { if (node.IsFile()) { RescanFile(directive, node.AsFile()); } else { Log.Debug("Directive {}: ignoring non-directive, non-file inode {} ", directive.GetId (), node); } } } } }
/// <summary> /// Test that an append with no locations fails with an exception /// showing insufficient locations. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestAppendInsufficientLocations() { Configuration conf = new Configuration(); // lower heartbeat interval for fast recognition of DN conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 3000); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build(); DistributedFileSystem fileSystem = null; try { // create a file with replication 3 fileSystem = cluster.GetFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.Create(f, (short)2); create.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); create.Close(); // Check for replications DFSTestUtil.WaitReplication(fileSystem, f, (short)2); // Shut down all DNs that have the last block location for the file LocatedBlocks lbs = fileSystem.dfs.GetNamenode().GetBlockLocations("/testAppend", 0, long.MaxValue); IList <DataNode> dnsOfCluster = cluster.GetDataNodes(); DatanodeInfo[] dnsWithLocations = lbs.GetLastLocatedBlock().GetLocations(); foreach (DataNode dn in dnsOfCluster) { foreach (DatanodeInfo loc in dnsWithLocations) { if (dn.GetDatanodeId().Equals(loc)) { dn.Shutdown(); DFSTestUtil.WaitForDatanodeDeath(dn); } } } // Wait till 0 replication is recognized DFSTestUtil.WaitReplication(fileSystem, f, (short)0); // Append to the file, at this state there are 3 live DNs but none of them // have the block. try { fileSystem.Append(f); NUnit.Framework.Assert.Fail("Append should fail because insufficient locations"); } catch (IOException e) { Log.Info("Expected exception: ", e); } FSDirectory dir = cluster.GetNamesystem().GetFSDirectory(); INodeFile inode = INodeFile.ValueOf(dir.GetINode("/testAppend"), "/testAppend"); NUnit.Framework.Assert.IsTrue("File should remain closed", !inode.IsUnderConstruction ()); } finally { if (null != fileSystem) { fileSystem.Close(); } cluster.Shutdown(); } }