internal virtual void Validate() { // check parent up to snapshotRootIndex if this is a snapshot path int i = 0; if (inodes[i] != null) { for (i++; i < inodes.Length && inodes[i] != null; i++) { INodeDirectory parent_i = inodes[i].GetParent(); INodeDirectory parent_i_1 = inodes[i - 1].GetParent(); if (parent_i != inodes[i - 1] && (parent_i_1 == null || !parent_i_1.IsSnapshottable () || parent_i != parent_i_1)) { throw new Exception("inodes[" + i + "].getParent() != inodes[" + (i - 1) + "]\n inodes[" + i + "]=" + inodes[i].ToDetailString() + "\n inodes[" + (i - 1) + "]=" + inodes [i - 1].ToDetailString() + "\n this=" + ToString(false)); } } } if (i != inodes.Length) { throw new Exception("i = " + i + " != " + inodes.Length + ", this=" + ToString(false )); } }
private void LoadRootINode(FsImageProto.INodeSection.INode p) { INodeDirectory root = LoadINodeDirectory(p, parent.GetLoaderContext()); QuotaCounts q = root.GetQuotaCounts(); long nsQuota = q.GetNameSpace(); long dsQuota = q.GetStorageSpace(); if (nsQuota != -1 || dsQuota != -1) { dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(nsQuota, dsQuota); } EnumCounters <StorageType> typeQuotas = q.GetTypeSpaces(); if (typeQuotas.AnyGreaterOrEqual(0)) { dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(typeQuotas); } dir.rootDir.CloneModificationTime(root); dir.rootDir.ClonePermissionStatus(root); AclFeature af = root.GetFeature(typeof(AclFeature)); if (af != null) { dir.rootDir.AddAclFeature(af); } // root dir supports having extended attributes according to POSIX XAttrFeature f = root.GetXAttrFeature(); if (f != null) { dir.rootDir.AddXAttrFeature(f); } dir.AddRootDirToEncryptionZone(f); }
/// <summary>Get a listing of all the snapshots of a snapshottable directory</summary> /// <exception cref="System.IO.IOException"/> private static DirectoryListing GetSnapshotsListing(FSDirectory fsd, string src, byte[] startAfter) { Preconditions.CheckState(fsd.HasReadLock()); Preconditions.CheckArgument(src.EndsWith(HdfsConstants.SeparatorDotSnapshotDir), "%s does not end with %s", src, HdfsConstants.SeparatorDotSnapshotDir); string dirPath = FSDirectory.NormalizePath(Sharpen.Runtime.Substring(src, 0, src. Length - HdfsConstants.DotSnapshotDir.Length)); INode node = fsd.GetINode(dirPath); INodeDirectory dirNode = INodeDirectory.ValueOf(node, dirPath); DirectorySnapshottableFeature sf = dirNode.GetDirectorySnapshottableFeature(); if (sf == null) { throw new SnapshotException("Directory is not a snapshottable directory: " + dirPath ); } ReadOnlyList <Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot> snapshots = sf.GetSnapshotList(); int skipSize = ReadOnlyList.Util.BinarySearch(snapshots, startAfter); skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1; int numOfListing = Math.Min(snapshots.Size() - skipSize, fsd.GetLsLimit()); HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing; i++) { Snapshot.Root sRoot = snapshots.Get(i + skipSize).GetRoot(); listing[i] = CreateFileStatus(fsd, src, sRoot.GetLocalNameBytes(), sRoot, BlockStoragePolicySuite .IdUnspecified, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId , false, INodesInPath.FromINode(sRoot)); } return(new DirectoryListing(listing, snapshots.Size() - skipSize - numOfListing)); }
/// <summary> /// Guarded by /// <see cref="FSNamesystem.ReadLock()"/> /// /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> private void CheckSubAccess(byte[][] pathByNameArr, int pathIdx, INode inode, int snapshotId, FsAction access, bool ignoreEmptyDir) { if (inode == null || !inode.IsDirectory()) { return; } Stack <INodeDirectory> directories = new Stack <INodeDirectory>(); for (directories.Push(inode.AsDirectory()); !directories.IsEmpty();) { INodeDirectory d = directories.Pop(); ReadOnlyList <INode> cList = d.GetChildrenList(snapshotId); if (!(cList.IsEmpty() && ignoreEmptyDir)) { //TODO have to figure this out with inodeattribute provider Check(GetINodeAttrs(pathByNameArr, pathIdx, d, snapshotId), inode.GetFullPathName (), access); } foreach (INode child in cList) { if (child.IsDirectory()) { directories.Push(child.AsDirectory()); } } } }
/// <summary>Set the namespace, storagespace and typespace quota for a directory.</summary> /// <remarks> /// Set the namespace, storagespace and typespace quota for a directory. /// Note: This does not support ".inodes" relative path. /// </remarks> /// <exception cref="System.IO.IOException"/> internal static void SetQuota(FSDirectory fsd, string src, long nsQuota, long ssQuota , StorageType type) { if (fsd.IsPermissionEnabled()) { FSPermissionChecker pc = fsd.GetPermissionChecker(); pc.CheckSuperuserPrivilege(); } fsd.WriteLock(); try { INodeDirectory changed = UnprotectedSetQuota(fsd, src, nsQuota, ssQuota, type); if (changed != null) { QuotaCounts q = changed.GetQuotaCounts(); if (type == null) { fsd.GetEditLog().LogSetQuota(src, q.GetNameSpace(), q.GetStorageSpace()); } else { fsd.GetEditLog().LogSetQuotaByStorageType(src, q.GetTypeSpaces().Get(type), type); } } } finally { fsd.WriteUnlock(); } }
/// <summary> /// Serialize a /// <see cref="INodeDirectory"/> /// </summary> /// <param name="node">The node to write</param> /// <param name="out"> /// The /// <see cref="System.IO.DataOutput"/> /// where the fields are written /// </param> /// <exception cref="System.IO.IOException"/> public static void WriteINodeDirectory(INodeDirectory node, DataOutput @out) { WriteLocalName(node, @out); @out.WriteLong(node.GetId()); @out.WriteShort(0); // replication @out.WriteLong(node.GetModificationTime()); @out.WriteLong(0); // access time @out.WriteLong(0); // preferred block size @out.WriteInt(-1); // # of blocks WriteQuota(node.GetQuotaCounts(), @out); if (node.IsSnapshottable()) { @out.WriteBoolean(true); } else { @out.WriteBoolean(false); @out.WriteBoolean(node.IsWithSnapshot()); } WritePermissionStatus(node, @out); }
/// <summary>Is this inode in the latest snapshot?</summary> public bool IsInLatestSnapshot(int latestSnapshotId) { if (latestSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot. CurrentStateId || latestSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .NoSnapshotId) { return(false); } // if parent is a reference node, parent must be a renamed node. We can // stop the check at the reference node. if (parent != null && parent.IsReference()) { return(true); } INodeDirectory parentDir = GetParent(); if (parentDir == null) { // root return(true); } if (!parentDir.IsInLatestSnapshot(latestSnapshotId)) { return(false); } Org.Apache.Hadoop.Hdfs.Server.Namenode.INode child = parentDir.GetChild(GetLocalNameBytes (), latestSnapshotId); if (this == child) { return(true); } return(child != null && child.IsReference() && this == child.AsReference().GetReferredINode ()); }
/// <exception cref="System.IO.IOException"/> internal void LoadINodeDirectorySection(InputStream @in) { IList <INodeReference> refList = parent.GetLoaderContext().GetRefList(); while (true) { FsImageProto.INodeDirectorySection.DirEntry e = FsImageProto.INodeDirectorySection.DirEntry .ParseDelimitedFrom(@in); // note that in is a LimitedInputStream if (e == null) { break; } INodeDirectory p = dir.GetInode(e.GetParent()).AsDirectory(); foreach (long id in e.GetChildrenList()) { INode child = dir.GetInode(id); AddToParent(p, child); } foreach (int refId in e.GetRefChildrenList()) { INodeReference @ref = refList[refId]; AddToParent(p, @ref); } } }
public virtual void TestAclGroupTraverseDenyOnlyDefaultEntries() { INodeDirectory inodeDir = CreateINodeDirectory(inodeRoot, "dir1", "bruce", "execs" , (short)0x1ed); INodeFile inodeFile = CreateINodeFile(inodeDir, "file1", "bruce", "execs", (short )0x1a4); AddAcl(inodeDir, AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.User, FsAction.All), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.Group, FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.Other , FsAction.ReadExecute), AclTestHelpers.AclEntry(AclEntryScope.Default, AclEntryType .User, FsAction.All), AclTestHelpers.AclEntry(AclEntryScope.Default, AclEntryType .Group, "sales", FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Default, AclEntryType.Group, FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Default , AclEntryType.Other, FsAction.ReadExecute)); AssertPermissionGranted(Bruce, "/dir1/file1", FsAction.ReadWrite); AssertPermissionGranted(Diana, "/dir1/file1", FsAction.Read); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Read); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Write); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Execute); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.ReadWrite); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.ReadExecute); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.WriteExecute); AssertPermissionDenied(Clark, "/dir1/file1", FsAction.All); }
/// <summary> /// Check if the given INode (or one of its descendants) is snapshottable and /// already has snapshots. /// </summary> /// <param name="target">The given INode</param> /// <param name="snapshottableDirs"> /// The list of directories that are snapshottable /// but do not have snapshots yet /// </param> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/> internal static void CheckSnapshot(INode target, IList <INodeDirectory> snapshottableDirs ) { if (target.IsDirectory()) { INodeDirectory targetDir = target.AsDirectory(); DirectorySnapshottableFeature sf = targetDir.GetDirectorySnapshottableFeature(); if (sf != null) { if (sf.GetNumSnapshots() > 0) { string fullPath = targetDir.GetFullPathName(); throw new SnapshotException("The directory " + fullPath + " cannot be deleted since " + fullPath + " is snapshottable and already has snapshots"); } else { if (snapshottableDirs != null) { snapshottableDirs.AddItem(targetDir); } } } foreach (INode child in targetDir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId)) { CheckSnapshot(child, snapshottableDirs); } } }
public DstReference(INodeDirectory parent, INodeReference.WithCount referred, int dstSnapshotId) : base(parent, referred) { this.dstSnapshotId = dstSnapshotId; referred.AddReference(this); }
private void CheckStoragespace(INodeDirectory dir, long computed) { if (-1 != quota.GetStorageSpace() && usage.GetStorageSpace() != computed) { NameNode.Log.Error("BUG: Inconsistent storagespace for directory " + dir.GetFullPathName () + ". Cached = " + usage.GetStorageSpace() + " != Computed = " + computed); } }
public WithName(INodeDirectory parent, INodeReference.WithCount referred, byte[] name, int lastSnapshotId) : base(parent, referred) { this.name = name; this.lastSnapshotId = lastSnapshotId; referred.AddReference(this); }
public CopyWithQuota(INodeDirectory dir) : base(dir) { Preconditions.CheckArgument(dir.IsQuotaSet()); QuotaCounts q = dir.GetQuotaCounts(); this.quota = new QuotaCounts.Builder().QuotaCount(q).Build(); }
internal static void AddFiles(FSEditLog editLog, int numFiles, short replication, int blocksPerFile, long startingBlockId, long blockSize, FileNameGenerator nameGenerator ) { PermissionStatus p = new PermissionStatus("joeDoe", "people", new FsPermission((short )0x1ff)); INodeId inodeId = new INodeId(); INodeDirectory dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L); editLog.LogMkDir(BasePath, dirInode); BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile]; for (int iB = 0; iB < blocksPerFile; ++iB) { blocks[iB] = new BlockInfoContiguous(new Block(0, blockSize, BlockGenerationStamp ), replication); } long currentBlockId = startingBlockId; long bidAtSync = startingBlockId; for (int iF = 0; iF < numFiles; iF++) { for (int iB_1 = 0; iB_1 < blocksPerFile; ++iB_1) { blocks[iB_1].SetBlockId(currentBlockId++); } INodeFile inode = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, blocks, replication , blockSize, unchecked ((byte)0)); inode.ToUnderConstruction(string.Empty, string.Empty); // Append path to filename with information about blockIDs string path = "_" + iF + "_B" + blocks[0].GetBlockId() + "_to_B" + blocks[blocksPerFile - 1].GetBlockId() + "_"; string filePath = nameGenerator.GetNextFileName(string.Empty); filePath = filePath + path; // Log the new sub directory in edits if ((iF % nameGenerator.GetFilesPerDirectory()) == 0) { string currentDir = nameGenerator.GetCurrentDir(); dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L); editLog.LogMkDir(currentDir, dirInode); } INodeFile fileUc = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, BlockInfoContiguous .EmptyArray, replication, blockSize); fileUc.ToUnderConstruction(string.Empty, string.Empty); editLog.LogOpenFile(filePath, fileUc, false, false); editLog.LogCloseFile(filePath, inode); if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks editLog.LogSync(); bidAtSync = currentBlockId; } } System.Console.Out.WriteLine("Created edits log in directory " + edits_dir); System.Console.Out.WriteLine(" containing " + numFiles + " File-Creates, each file with " + blocksPerFile + " blocks"); System.Console.Out.WriteLine(" blocks range: " + startingBlockId + " to " + (currentBlockId - 1)); }
/// <exception cref="System.IO.IOException"/> private void Save(OutputStream @out, INodeDirectory n) { FsImageProto.INodeSection.INodeDirectory.Builder b = BuildINodeDirectory(n, parent .GetSaverContext()); FsImageProto.INodeSection.INode r = ((FsImageProto.INodeSection.INode)BuildINodeCommon (n).SetType(FsImageProto.INodeSection.INode.Type.Directory).SetDirectory(b).Build ()); r.WriteDelimitedTo(@out); }
internal override void RecordModification(int latestSnapshotId) { if (IsInLatestSnapshot(latestSnapshotId)) { INodeDirectory parent = GetParent(); parent.SaveChild2Snapshot(this, latestSnapshotId, new Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeSymlink (this)); } }
public virtual void SetUp() { Configuration conf = new Configuration(); FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>(); Org.Mockito.Mockito.DoAnswer(new _Answer_81()).When(fsn).CreateFsOwnerPermissions (Matchers.Any <FsPermission>()); dir = new FSDirectory(fsn, conf); inodeRoot = dir.GetRoot(); }
/// <exception cref="System.IO.IOException"/> private static void SetDirStoragePolicy(FSDirectory fsd, INodeDirectory inode, byte policyId, int latestSnapshotId) { IList <XAttr> existingXAttrs = XAttrStorage.ReadINodeXAttrs(inode); XAttr xAttr = BlockStoragePolicySuite.BuildXAttr(policyId); IList <XAttr> newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsd, existingXAttrs, Arrays. AsList(xAttr), EnumSet.Of(XAttrSetFlag.Create, XAttrSetFlag.Replace)); XAttrStorage.UpdateINodeXAttrs(inode, newXAttrs, latestSnapshotId); }
/// <exception cref="System.IO.IOException"/> private static INodeDirectory CreateINodeDirectory(INodeDirectory parent, string name, string owner, string group, short perm) { PermissionStatus permStatus = PermissionStatus.CreateImmutable(owner, group, FsPermission .CreateImmutable(perm)); INodeDirectory inodeDirectory = new INodeDirectory(INodeId.GrandfatherInodeId, Sharpen.Runtime.GetBytesForString (name, "UTF-8"), permStatus, 0L); parent.AddChild(inodeDirectory); return(inodeDirectory); }
/// <returns>true if the given inode is an ancestor directory of this inode.</returns> public bool IsAncestorDirectory(INodeDirectory dir) { for (INodeDirectory p = GetParent(); p != null; p = p.GetParent()) { if (p == dir) { return(true); } } return(false); }
internal static Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeMap NewInstance(INodeDirectory rootDir) { // Compute the map capacity by allocating 1% of total memory int capacity = LightWeightGSet.ComputeCapacity(1, "INodeMap"); GSet <INode, INodeWithAdditionalFields> map = new LightWeightGSet <INode, INodeWithAdditionalFields >(capacity); map.Put(rootDir); return(new Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeMap(map)); }
/// <exception cref="System.IO.IOException"/> private static INodeFile CreateINodeFile(INodeDirectory parent, string name, string owner, string group, short perm) { PermissionStatus permStatus = PermissionStatus.CreateImmutable(owner, group, FsPermission .CreateImmutable(perm)); INodeFile inodeFile = new INodeFile(INodeId.GrandfatherInodeId, Sharpen.Runtime.GetBytesForString (name, "UTF-8"), permStatus, 0L, 0L, null, Replication, PreferredBlockSize, unchecked ( (byte)0)); parent.AddChild(inodeFile); return(inodeFile); }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> internal RenameOperation(FSDirectory fsd, string src, string dst, INodesInPath srcIIP , INodesInPath dstIIP) { this.fsd = fsd; this.src = src; this.dst = dst; this.srcIIP = srcIIP; this.dstIIP = dstIIP; this.srcParentIIP = srcIIP.GetParentINodesInPath(); this.dstParentIIP = dstIIP.GetParentINodesInPath(); BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite(); srcChild = this.srcIIP.GetLastINode(); srcChildName = srcChild.GetLocalNameBytes(); int srcLatestSnapshotId = srcIIP.GetLatestSnapshotId(); isSrcInSnapshot = srcChild.IsInLatestSnapshot(srcLatestSnapshotId); srcChildIsReference = srcChild.IsReference(); srcParent = this.srcIIP.GetINode(-2).AsDirectory(); // Record the snapshot on srcChild. After the rename, before any new // snapshot is taken on the dst tree, changes will be recorded in the // latest snapshot of the src tree. if (isSrcInSnapshot) { srcChild.RecordModification(srcLatestSnapshotId); } // check srcChild for reference srcRefDstSnapshot = srcChildIsReference ? srcChild.AsReference().GetDstSnapshotId () : Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId; oldSrcCounts = new QuotaCounts.Builder().Build(); if (isSrcInSnapshot) { INodeReference.WithName withName = srcParent.ReplaceChild4ReferenceWithName(srcChild , srcLatestSnapshotId); withCount = (INodeReference.WithCount)withName.GetReferredINode(); srcChild = withName; this.srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, srcChild); // get the counts before rename withCount.GetReferredINode().ComputeQuotaUsage(bsps, oldSrcCounts, true); } else { if (srcChildIsReference) { // srcChild is reference but srcChild is not in latest snapshot withCount = (INodeReference.WithCount)srcChild.AsReference().GetReferredINode(); } else { withCount = null; } } }
/// <summary> /// <inheritDoc/> /// <br/> /// To destroy a DstReference node, we first remove its link with the /// referred node. If the reference number of the referred node is <= 0, we /// destroy the subtree of the referred node. Otherwise, we clean the /// referred node's subtree and delete everything created after the last /// rename operation, i.e., everything outside of the scope of the prior /// WithName nodes. /// </summary> public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes) { if (RemoveReference(this) <= 0) { GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); } else { // we will clean everything, including files, directories, and // snapshots, that were created after this prior snapshot int prior = GetPriorSnapshot(this); // prior must be non-null, otherwise we do not have any previous // WithName nodes, and the reference number will be 0. Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .NoSnapshotId); // identify the snapshot created after prior int snapshot = GetSelfSnapshot(prior); INode referred = GetReferredINode().AsReference().GetReferredINode(); if (referred.IsFile()) { // if referred is a file, it must be a file with snapshot since we did // recordModification before the rename INodeFile file = referred.AsFile(); Preconditions.CheckState(file.IsWithSnapshot()); // make sure we mark the file as deleted file.GetFileWithSnapshotFeature().DeleteCurrentFile(); // when calling cleanSubtree of the referred node, since we // compute quota usage updates before calling this destroy // function, we use true for countDiffChange referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes); } else { if (referred.IsDirectory()) { // similarly, if referred is a directory, it must be an // INodeDirectory with snapshot INodeDirectory dir = referred.AsDirectory(); Preconditions.CheckState(dir.IsWithSnapshot()); try { DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks , removedINodes); } catch (QuotaExceededException e) { Log.Error("should not exceed quota while snapshot deletion", e); } } } } }
public virtual void TestSnapshotOnRoot() { Path root = new Path("/"); hdfs.AllowSnapshot(root); hdfs.CreateSnapshot(root, "s1"); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); // save namespace and restart cluster hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); INodeDirectory rootNode = fsn.dir.GetINode4Write(root.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue("The children list of root should be empty", rootNode .GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ).IsEmpty()); // one snapshot on root: s1 IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = rootNode.GetDiffs(). AsList(); NUnit.Framework.Assert.AreEqual(1, diffList.Count); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s1 = rootNode.GetSnapshot (DFSUtil.String2Bytes("s1")); NUnit.Framework.Assert.AreEqual(s1.GetId(), diffList[0].GetSnapshotId()); // check SnapshotManager's snapshottable directory list NUnit.Framework.Assert.AreEqual(1, fsn.GetSnapshotManager().GetNumSnapshottableDirs ()); SnapshottableDirectoryStatus[] sdirs = fsn.GetSnapshotManager().GetSnapshottableDirListing (null); NUnit.Framework.Assert.AreEqual(root, sdirs[0].GetFullPath()); // save namespace and restart cluster hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); }
/// <summary> /// Delete a path from the name space /// Update the count at each ancestor directory with quota /// </summary> /// <param name="iip">the inodes resolved from the path</param> /// <param name="collectedBlocks">blocks collected from the deleted path</param> /// <param name="removedINodes">inodes that should be removed from inodeMap</param> /// <param name="mtime">the time the inode is removed</param> /// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns> private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes, long mtime) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); // check if target node exists INode targetNode = iip.GetLastINode(); if (targetNode == null) { return(-1); } // record modification int latestSnapshot = iip.GetLatestSnapshotId(); targetNode.RecordModification(latestSnapshot); // Remove the node from the namespace long removed = fsd.RemoveLastINode(iip); if (removed == -1) { return(-1); } // set the parent's modification time INodeDirectory parent = targetNode.GetParent(); parent.UpdateModificationTime(mtime, latestSnapshot); fsd.UpdateCountForDelete(targetNode, iip); if (removed == 0) { return(0); } // collect block and update quota if (!targetNode.IsInLatestSnapshot(latestSnapshot)) { targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks , removedINodes); } else { QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId, latestSnapshot, collectedBlocks, removedINodes); removed = counts.GetNameSpace(); fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation()); } if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath () + " is removed"); } return(removed); }
internal ContentSummaryComputationContext ComputeContentSummary(INodeDirectory dir , ContentSummaryComputationContext summary) { long original = summary.GetCounts().GetStoragespace(); long oldYieldCount = summary.GetYieldCount(); dir.ComputeDirectoryContentSummary(summary, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId); // Check only when the content has not changed in the middle. if (oldYieldCount == summary.GetYieldCount()) { CheckStoragespace(dir, summary.GetCounts().GetStoragespace() - original); } return(summary); }
/// <summary>Test if the quota can be correctly updated for append</summary> /// <exception cref="System.Exception"/> public virtual void TestUpdateQuotaForAppend() { Path foo = new Path(dir, "foo"); Path bar = new Path(foo, "bar"); long currentFileLen = Blocksize; DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed); dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1); // append half of the block data, the previous file length is at block // boundary DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2); currentFileLen += (Blocksize / 2); INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory(); NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet()); QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); long ns = quota.GetNameSpace(); long ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); // foo and bar NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds); ContentSummary c = dfs.GetContentSummary(foo); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds); // append another block, the previous file length is not at block boundary DFSTestUtil.AppendFile(dfs, bar, Blocksize); currentFileLen += Blocksize; quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); ns = quota.GetNameSpace(); ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); // foo and bar NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds); c = dfs.GetContentSummary(foo); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds); // append several blocks DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8); currentFileLen += (Blocksize * 3 + Blocksize / 8); quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); ns = quota.GetNameSpace(); ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); // foo and bar NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds); c = dfs.GetContentSummary(foo); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds); }
private void AddToParent(INodeDirectory parent, INode child) { if (parent == dir.rootDir && FSDirectory.IsReservedName(child)) { throw new HadoopIllegalArgumentException("File name \"" + child.GetLocalName() + "\" is reserved. Please " + " change the name of the existing file or directory to another " + "name before upgrading to this release."); } // NOTE: This does not update space counts for parents if (!parent.AddChild(child)) { return; } dir.CacheName(child); if (child.IsFile()) { UpdateBlocksMap(child.AsFile(), fsn.GetBlockManager()); } }