/// <summary> /// Check if the given INode (or one of its descendants) is snapshottable and /// already has snapshots. /// </summary> /// <param name="target">The given INode</param> /// <param name="snapshottableDirs"> /// The list of directories that are snapshottable /// but do not have snapshots yet /// </param> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/> internal static void CheckSnapshot(INode target, IList <INodeDirectory> snapshottableDirs ) { if (target.IsDirectory()) { INodeDirectory targetDir = target.AsDirectory(); DirectorySnapshottableFeature sf = targetDir.GetDirectorySnapshottableFeature(); if (sf != null) { if (sf.GetNumSnapshots() > 0) { string fullPath = targetDir.GetFullPathName(); throw new SnapshotException("The directory " + fullPath + " cannot be deleted since " + fullPath + " is snapshottable and already has snapshots"); } else { if (snapshottableDirs != null) { snapshottableDirs.AddItem(targetDir); } } } foreach (INode child in targetDir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId)) { CheckSnapshot(child, snapshottableDirs); } } }
/// <summary> /// Guarded by /// <see cref="FSNamesystem.ReadLock()"/> /// /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> private void CheckSubAccess(byte[][] pathByNameArr, int pathIdx, INode inode, int snapshotId, FsAction access, bool ignoreEmptyDir) { if (inode == null || !inode.IsDirectory()) { return; } Stack <INodeDirectory> directories = new Stack <INodeDirectory>(); for (directories.Push(inode.AsDirectory()); !directories.IsEmpty();) { INodeDirectory d = directories.Pop(); ReadOnlyList <INode> cList = d.GetChildrenList(snapshotId); if (!(cList.IsEmpty() && ignoreEmptyDir)) { //TODO have to figure this out with inodeattribute provider Check(GetINodeAttrs(pathByNameArr, pathIdx, d, snapshotId), inode.GetFullPathName (), access); } foreach (INode child in cList) { if (child.IsDirectory()) { directories.Push(child.AsDirectory()); } } } }
public virtual void TestReset() { Configuration conf = new Configuration(); FSEditLog fsEditLog = Org.Mockito.Mockito.Mock <FSEditLog>(); FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>(); Org.Mockito.Mockito.When(fsImage.GetEditLog()).ThenReturn(fsEditLog); FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsn.ImageLoadComplete(); NUnit.Framework.Assert.IsTrue(fsn.IsImageLoaded()); fsn.Clear(); NUnit.Framework.Assert.IsFalse(fsn.IsImageLoaded()); INodeDirectory root = (INodeDirectory)fsn.GetFSDirectory().GetINode("/"); NUnit.Framework.Assert.IsTrue(root.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId).IsEmpty()); fsn.ImageLoadComplete(); NUnit.Framework.Assert.IsTrue(fsn.IsImageLoaded()); }
public virtual void TestSnapshotOnRoot() { Path root = new Path("/"); hdfs.AllowSnapshot(root); hdfs.CreateSnapshot(root, "s1"); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); // save namespace and restart cluster hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); INodeDirectory rootNode = fsn.dir.GetINode4Write(root.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue("The children list of root should be empty", rootNode .GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ).IsEmpty()); // one snapshot on root: s1 IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = rootNode.GetDiffs(). AsList(); NUnit.Framework.Assert.AreEqual(1, diffList.Count); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s1 = rootNode.GetSnapshot (DFSUtil.String2Bytes("s1")); NUnit.Framework.Assert.AreEqual(s1.GetId(), diffList[0].GetSnapshotId()); // check SnapshotManager's snapshottable directory list NUnit.Framework.Assert.AreEqual(1, fsn.GetSnapshotManager().GetNumSnapshottableDirs ()); SnapshottableDirectoryStatus[] sdirs = fsn.GetSnapshotManager().GetSnapshottableDirListing (null); NUnit.Framework.Assert.AreEqual(root, sdirs[0].GetFullPath()); // save namespace and restart cluster hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); }
/// <summary> /// Get a partial listing of the indicated directory /// We will stop when any of the following conditions is met: /// 1) this.lsLimit files have been added /// 2) needLocation is true AND enough files have been added such /// that at least this.lsLimit block locations are in the response /// </summary> /// <param name="fsd">FSDirectory</param> /// <param name="iip"> /// the INodesInPath instance containing all the INodes along the /// path /// </param> /// <param name="src">the directory name</param> /// <param name="startAfter">the name to start listing after</param> /// <param name="needLocation">if block locations are returned</param> /// <returns>a partial listing starting after startAfter</returns> /// <exception cref="System.IO.IOException"/> private static DirectoryListing GetListing(FSDirectory fsd, INodesInPath iip, string src, byte[] startAfter, bool needLocation, bool isSuperUser) { string srcs = FSDirectory.NormalizePath(src); bool isRawPath = FSDirectory.IsReservedRawName(src); fsd.ReadLock(); try { if (srcs.EndsWith(HdfsConstants.SeparatorDotSnapshotDir)) { return(GetSnapshotsListing(fsd, srcs, startAfter)); } int snapshot = iip.GetPathSnapshotId(); INode targetNode = iip.GetLastINode(); if (targetNode == null) { return(null); } byte parentStoragePolicy = isSuperUser ? targetNode.GetStoragePolicyID() : BlockStoragePolicySuite .IdUnspecified; if (!targetNode.IsDirectory()) { return(new DirectoryListing(new HdfsFileStatus[] { CreateFileStatus(fsd, src, HdfsFileStatus .EmptyName, targetNode, needLocation, parentStoragePolicy, snapshot, isRawPath, iip) }, 0)); } INodeDirectory dirInode = targetNode.AsDirectory(); ReadOnlyList <INode> contents = dirInode.GetChildrenList(snapshot); int startChild = INodeDirectory.NextChild(contents, startAfter); int totalNumChildren = contents.Size(); int numOfListing = Math.Min(totalNumChildren - startChild, fsd.GetLsLimit()); int locationBudget = fsd.GetLsLimit(); int listingCnt = 0; HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing && locationBudget > 0; i++) { INode cur = contents.Get(startChild + i); byte curPolicy = isSuperUser && !cur.IsSymlink() ? cur.GetLocalStoragePolicyID() : BlockStoragePolicySuite.IdUnspecified; listing[i] = CreateFileStatus(fsd, src, cur.GetLocalNameBytes(), cur, needLocation , GetStoragePolicyID(curPolicy, parentStoragePolicy), snapshot, isRawPath, iip); listingCnt++; if (needLocation) { // Once we hit lsLimit locations, stop. // This helps to prevent excessively large response payloads. // Approximate #locations with locatedBlockCount() * repl_factor LocatedBlocks blks = ((HdfsLocatedFileStatus)listing[i]).GetBlockLocations(); locationBudget -= (blks == null) ? 0 : blks.LocatedBlockCount() * listing[i].GetReplication (); } } // truncate return array if necessary if (listingCnt < numOfListing) { listing = Arrays.CopyOf(listing, listingCnt); } return(new DirectoryListing(listing, totalNumChildren - startChild - listingCnt)); } finally { fsd.ReadUnlock(); } }