/// <summary>for normal (non-snapshot) file.</summary> /// <exception cref="System.Exception"/> public virtual void TestNonSnapshotPathINodes() { // Get the inodes by resolving the path of a normal file string[] names = INode.GetPathNames(file1.ToString()); byte[][] components = INode.GetPathComponents(names); INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // The number of inodes should be equal to components.length NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length); // The returned nodesInPath should be non-snapshot AssertSnapshot(nodesInPath, false, null, -1); // The last INode should be associated with file1 NUnit.Framework.Assert.IsTrue("file1=" + file1 + ", nodesInPath=" + nodesInPath, nodesInPath.GetINode(components.Length - 1) != null); NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(components.Length - 1).GetFullPathName (), file1.ToString()); NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(components.Length - 2).GetFullPathName (), sub1.ToString()); NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(components.Length - 3).GetFullPathName (), dir.ToString()); nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length); AssertSnapshot(nodesInPath, false, null, -1); NUnit.Framework.Assert.AreEqual(nodesInPath.GetLastINode().GetFullPathName(), file1 .ToString()); }
/// <summary> /// For a given absolute path, create all ancestors as directories along the /// path. /// </summary> /// <remarks> /// For a given absolute path, create all ancestors as directories along the /// path. All ancestors inherit their parent's permission plus an implicit /// u+wx permission. This is used by create() and addSymlink() for /// implicitly creating all directories along the path. /// For example, path="/foo/bar/spam", "/foo" is an existing directory, /// "/foo/bar" is not existing yet, the function will create directory bar. /// </remarks> /// <returns> /// a tuple which contains both the new INodesInPath (with all the /// existing and newly created directories) and the last component in the /// relative path. Or return null if there are errors. /// </returns> /// <exception cref="System.IO.IOException"/> internal static KeyValuePair <INodesInPath, string> CreateAncestorDirectories(FSDirectory fsd, INodesInPath iip, PermissionStatus permission) { string last = new string(iip.GetLastLocalName(), Charsets.Utf8); INodesInPath existing = iip.GetExistingINodes(); IList <string> children = iip.GetPath(existing.Length(), iip.Length() - existing.Length ()); int size = children.Count; if (size > 1) { // otherwise all ancestors have been created IList <string> directories = children.SubList(0, size - 1); INode parentINode = existing.GetLastINode(); // Ensure that the user can traversal the path by adding implicit // u+wx permission to all ancestor directories existing = CreateChildrenDirectories(fsd, existing, directories, AddImplicitUwx(parentINode .GetPermissionStatus(), permission)); if (existing == null) { return(null); } } return(new AbstractMap.SimpleImmutableEntry <INodesInPath, string>(existing, last)); }
/// <summary>for snapshot file while adding a new file after snapshot.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotPathINodesWithAddedFile() { // Create a snapshot for the dir, and check the inodes for the path // pointing to a snapshot file hdfs.AllowSnapshot(sub1); hdfs.CreateSnapshot(sub1, "s4"); // Add a new file /TestSnapshot/sub1/file3 Path file3 = new Path(sub1, "file3"); DFSTestUtil.CreateFile(hdfs, file3, 1024, Replication, seed); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s4; { // Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3 string snapshotPath = sub1.ToString() + "/.snapshot/s4/file3"; string[] names = INode.GetPathNames(snapshotPath); byte[][] components = INode.GetPathComponents(names); INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore // ".snapshot" NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1); // The number of non-null inodes should be components.length - 2, since // snapshot of file3 does not exist NUnit.Framework.Assert.AreEqual(GetNumNonNull(nodesInPath), components.Length - 2 ); s4 = GetSnapshot(nodesInPath, "s4", 3); // SnapshotRootIndex should still be 3: {root, Testsnapshot, sub1, s4, null} AssertSnapshot(nodesInPath, true, s4, 3); // Check the last INode in inodes, which should be null NUnit.Framework.Assert.IsNull(nodesInPath.GetINode(nodesInPath.Length() - 1)); } // Check the inodes for /TestSnapshot/sub1/file3 string[] names_1 = INode.GetPathNames(file3.ToString()); byte[][] components_1 = INode.GetPathComponents(names_1); INodesInPath nodesInPath_1 = INodesInPath.Resolve(fsdir.rootDir, components_1, false ); // The number of inodes should be equal to components.length NUnit.Framework.Assert.AreEqual(nodesInPath_1.Length(), components_1.Length); // The returned nodesInPath should be non-snapshot AssertSnapshot(nodesInPath_1, false, s4, -1); // The last INode should be associated with file3 NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 1).GetFullPathName (), file3.ToString()); NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 2).GetFullPathName (), sub1.ToString()); NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 3).GetFullPathName (), dir.ToString()); hdfs.DeleteSnapshot(sub1, "s4"); hdfs.DisallowSnapshot(sub1); }
/// <summary> /// Verify quota for rename operation where srcInodes[srcInodes.length-1] moves /// dstInodes[dstInodes.length-1] /// </summary> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> private static void VerifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath dst) { if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks()) { // Do not check quota if edits log is still being processed return; } int i = 0; while (src.GetINode(i) == dst.GetINode(i)) { i++; } // src[i - 1] is the last common ancestor. BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite(); QuotaCounts delta = src.GetLastINode().ComputeQuotaUsage(bsps); // Reduce the required quota by dst that is being removed INode dstINode = dst.GetLastINode(); if (dstINode != null) { delta.Subtract(dstINode.ComputeQuotaUsage(bsps)); } FSDirectory.VerifyQuota(dst, dst.Length() - 1, delta, src.GetINode(i - 1)); }
/// <exception cref="System.IO.IOException"/> internal virtual long RemoveSrc() { long removedNum = fsd.RemoveLastINode(srcIIP); if (removedNum == -1) { string error = "Failed to rename " + src + " to " + dst + " because the source can not be removed"; NameNode.stateChangeLog.Warn("DIR* FSDirRenameOp.unprotectedRenameTo:" + error); throw new IOException(error); } else { // update the quota count if necessary fsd.UpdateCountForDelete(srcChild, srcIIP); srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, null); return(removedNum); } }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> private static void VerifyQuota(FSDirectory fsd, INodesInPath targetIIP, QuotaCounts deltas) { if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks()) { // Do not check quota if editlog is still being processed return; } FSDirectory.VerifyQuota(targetIIP, targetIIP.Length() - 1, deltas, null); }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> internal RenameOperation(FSDirectory fsd, string src, string dst, INodesInPath srcIIP , INodesInPath dstIIP) { this.fsd = fsd; this.src = src; this.dst = dst; this.srcIIP = srcIIP; this.dstIIP = dstIIP; this.srcParentIIP = srcIIP.GetParentINodesInPath(); this.dstParentIIP = dstIIP.GetParentINodesInPath(); BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite(); srcChild = this.srcIIP.GetLastINode(); srcChildName = srcChild.GetLocalNameBytes(); int srcLatestSnapshotId = srcIIP.GetLatestSnapshotId(); isSrcInSnapshot = srcChild.IsInLatestSnapshot(srcLatestSnapshotId); srcChildIsReference = srcChild.IsReference(); srcParent = this.srcIIP.GetINode(-2).AsDirectory(); // Record the snapshot on srcChild. After the rename, before any new // snapshot is taken on the dst tree, changes will be recorded in the // latest snapshot of the src tree. if (isSrcInSnapshot) { srcChild.RecordModification(srcLatestSnapshotId); } // check srcChild for reference srcRefDstSnapshot = srcChildIsReference ? srcChild.AsReference().GetDstSnapshotId () : Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId; oldSrcCounts = new QuotaCounts.Builder().Build(); if (isSrcInSnapshot) { INodeReference.WithName withName = srcParent.ReplaceChild4ReferenceWithName(srcChild , srcLatestSnapshotId); withCount = (INodeReference.WithCount)withName.GetReferredINode(); srcChild = withName; this.srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, srcChild); // get the counts before rename withCount.GetReferredINode().ComputeQuotaUsage(bsps, oldSrcCounts, true); } else { if (srcChildIsReference) { // srcChild is reference but srcChild is not in latest snapshot withCount = (INodeReference.WithCount)srcChild.AsReference().GetReferredINode(); } else { withCount = null; } } }
internal virtual long RemoveDst() { long removedNum = fsd.RemoveLastINode(dstIIP); if (removedNum != -1) { oldDstChild = dstIIP.GetLastINode(); // update the quota count if necessary fsd.UpdateCountForDelete(oldDstChild, dstIIP); dstIIP = INodesInPath.Replace(dstIIP, dstIIP.Length() - 1, null); } return(removedNum); }
/// <summary> /// Delete a path from the name space /// Update the count at each ancestor directory with quota /// </summary> /// <param name="iip">the inodes resolved from the path</param> /// <param name="collectedBlocks">blocks collected from the deleted path</param> /// <param name="removedINodes">inodes that should be removed from inodeMap</param> /// <param name="mtime">the time the inode is removed</param> /// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns> private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes, long mtime) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); // check if target node exists INode targetNode = iip.GetLastINode(); if (targetNode == null) { return(-1); } // record modification int latestSnapshot = iip.GetLatestSnapshotId(); targetNode.RecordModification(latestSnapshot); // Remove the node from the namespace long removed = fsd.RemoveLastINode(iip); if (removed == -1) { return(-1); } // set the parent's modification time INodeDirectory parent = targetNode.GetParent(); parent.UpdateModificationTime(mtime, latestSnapshot); fsd.UpdateCountForDelete(targetNode, iip); if (removed == 0) { return(0); } // collect block and update quota if (!targetNode.IsInLatestSnapshot(latestSnapshot)) { targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks , removedINodes); } else { QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId, latestSnapshot, collectedBlocks, removedINodes); removed = counts.GetNameSpace(); fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation()); } if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath () + " is removed"); } return(removed); }
/// <summary>for snapshot file after deleting the original file.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotPathINodesAfterDeletion() { // Create a snapshot for the dir, and check the inodes for the path // pointing to a snapshot file hdfs.AllowSnapshot(sub1); hdfs.CreateSnapshot(sub1, "s2"); // Delete the original file /TestSnapshot/sub1/file1 hdfs.Delete(file1, false); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot; { // Resolve the path for the snapshot file // /TestSnapshot/sub1/.snapshot/s2/file1 string snapshotPath = sub1.ToString() + "/.snapshot/s2/file1"; string[] names = INode.GetPathNames(snapshotPath); byte[][] components = INode.GetPathComponents(names); INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore // ".snapshot" NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1); // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s2, file1} snapshot = GetSnapshot(nodesInPath, "s2", 3); AssertSnapshot(nodesInPath, true, snapshot, 3); // Check the INode for file1 (snapshot file) INode inode = nodesInPath.GetLastINode(); NUnit.Framework.Assert.AreEqual(file1.GetName(), inode.GetLocalName()); NUnit.Framework.Assert.IsTrue(inode.AsFile().IsWithSnapshot()); } // Check the INodes for path /TestSnapshot/sub1/file1 string[] names_1 = INode.GetPathNames(file1.ToString()); byte[][] components_1 = INode.GetPathComponents(names_1); INodesInPath nodesInPath_1 = INodesInPath.Resolve(fsdir.rootDir, components_1, false ); // The length of inodes should be equal to components.length NUnit.Framework.Assert.AreEqual(nodesInPath_1.Length(), components_1.Length); // The number of non-null elements should be components.length - 1 since // file1 has been deleted NUnit.Framework.Assert.AreEqual(GetNumNonNull(nodesInPath_1), components_1.Length - 1); // The returned nodesInPath should be non-snapshot AssertSnapshot(nodesInPath_1, false, snapshot, -1); // The last INode should be null, and the one before should be associated // with sub1 NUnit.Framework.Assert.IsNull(nodesInPath_1.GetINode(components_1.Length - 1)); NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 2).GetFullPathName (), sub1.ToString()); NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 3).GetFullPathName (), dir.ToString()); hdfs.DeleteSnapshot(sub1, "s2"); hdfs.DisallowSnapshot(sub1); }
private static bool DeleteAllowed(INodesInPath iip, string src) { if (iip.Length() < 1 || iip.GetLastINode() == null) { if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: failed to remove " + src + " because it does not exist"); } return(false); } else { if (iip.Length() == 1) { // src is the root NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedDelete: failed to remove " + src + " because the root is not allowed to be deleted"); return(false); } } return(true); }
/// <exception cref="System.IO.IOException"/> private static void ValidateRenameSource(INodesInPath srcIIP) { string error; INode srcInode = srcIIP.GetLastINode(); // validate source if (srcInode == null) { error = "rename source " + srcIIP.GetPath() + " is not found."; NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new FileNotFoundException(error); } if (srcIIP.Length() == 1) { error = "rename source cannot be the root"; NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new IOException(error); } // srcInode and its subtree cannot contain snapshottable directories with // snapshots FSDirSnapshotOp.CheckSnapshot(srcInode, null); }
/// <summary>Concat all the blocks from srcs to trg and delete the srcs files</summary> /// <param name="fsd">FSDirectory</param> /// <exception cref="System.IO.IOException"/> internal static void UnprotectedConcat(FSDirectory fsd, INodesInPath targetIIP, INodeFile [] srcList, long timestamp) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* FSNamesystem.concat to " + targetIIP.GetPath( )); } INodeFile trgInode = targetIIP.GetLastINode().AsFile(); QuotaCounts deltas = ComputeQuotaDeltas(fsd, trgInode, srcList); VerifyQuota(fsd, targetIIP, deltas); // the target file can be included in a snapshot trgInode.RecordModification(targetIIP.GetLatestSnapshotId()); INodeDirectory trgParent = targetIIP.GetINode(-2).AsDirectory(); trgInode.ConcatBlocks(srcList); // since we are in the same dir - we can use same parent to remove files int count = 0; foreach (INodeFile nodeToRemove in srcList) { if (nodeToRemove != null) { nodeToRemove.SetBlocks(null); nodeToRemove.GetParent().RemoveChild(nodeToRemove); fsd.GetINodeMap().Remove(nodeToRemove); count++; } } trgInode.SetModificationTime(timestamp, targetIIP.GetLatestSnapshotId()); trgParent.UpdateModificationTime(timestamp, targetIIP.GetLatestSnapshotId()); // update quota on the parent directory with deltas FSDirectory.UnprotectedUpdateCount(targetIIP, targetIIP.Length() - 1, deltas); }
/// <summary>for snapshot file while modifying file after snapshot.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotPathINodesAfterModification() { // First check the INode for /TestSnapshot/sub1/file1 string[] names = INode.GetPathNames(file1.ToString()); byte[][] components = INode.GetPathComponents(names); INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // The number of inodes should be equal to components.length NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length); // The last INode should be associated with file1 NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(components.Length - 1).GetFullPathName (), file1.ToString()); // record the modification time of the inode long modTime = nodesInPath.GetINode(nodesInPath.Length() - 1).GetModificationTime (); // Create a snapshot for the dir, and check the inodes for the path // pointing to a snapshot file hdfs.AllowSnapshot(sub1); hdfs.CreateSnapshot(sub1, "s3"); // Modify file1 DFSTestUtil.AppendFile(hdfs, file1, "the content for appending"); // Check the INodes for snapshot of file1 string snapshotPath = sub1.ToString() + "/.snapshot/s3/file1"; names = INode.GetPathNames(snapshotPath); components = INode.GetPathComponents(names); INodesInPath ssNodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false ); // Length of ssInodes should be (components.length - 1), since we will // ignore ".snapshot" NUnit.Framework.Assert.AreEqual(ssNodesInPath.Length(), components.Length - 1); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s3 = GetSnapshot(ssNodesInPath , "s3", 3); AssertSnapshot(ssNodesInPath, true, s3, 3); // Check the INode for snapshot of file1 INode snapshotFileNode = ssNodesInPath.GetLastINode(); NUnit.Framework.Assert.AreEqual(snapshotFileNode.GetLocalName(), file1.GetName()); NUnit.Framework.Assert.IsTrue(snapshotFileNode.AsFile().IsWithSnapshot()); // The modification time of the snapshot INode should be the same with the // original INode before modification NUnit.Framework.Assert.AreEqual(modTime, snapshotFileNode.GetModificationTime(ssNodesInPath .GetPathSnapshotId())); // Check the INode for /TestSnapshot/sub1/file1 again names = INode.GetPathNames(file1.ToString()); components = INode.GetPathComponents(names); INodesInPath newNodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false ); AssertSnapshot(newNodesInPath, false, s3, -1); // The number of inodes should be equal to components.length NUnit.Framework.Assert.AreEqual(newNodesInPath.Length(), components.Length); // The last INode should be associated with file1 int last = components.Length - 1; NUnit.Framework.Assert.AreEqual(newNodesInPath.GetINode(last).GetFullPathName(), file1.ToString()); // The modification time of the INode for file3 should have been changed NUnit.Framework.Assert.IsFalse(modTime == newNodesInPath.GetINode(last).GetModificationTime ()); hdfs.DeleteSnapshot(sub1, "s3"); hdfs.DisallowSnapshot(sub1); }
/// <summary>Rename src to dst.</summary> /// <remarks> /// Rename src to dst. /// See /// <see cref="Org.Apache.Hadoop.Hdfs.DistributedFileSystem.Rename(Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Options.Rename[]) /// "/> /// for details related to rename semantics and exceptions. /// </remarks> /// <param name="fsd">FSDirectory</param> /// <param name="src">source path</param> /// <param name="dst">destination path</param> /// <param name="timestamp">modification time</param> /// <param name="collectedBlocks">blocks to be removed</param> /// <param name="options">Rename options</param> /// <returns>whether a file/directory gets overwritten in the dst path</returns> /// <exception cref="System.IO.IOException"/> internal static bool UnprotectedRenameTo(FSDirectory fsd, string src, string dst, INodesInPath srcIIP, INodesInPath dstIIP, long timestamp, INode.BlocksMapUpdateInfo collectedBlocks, params Options.Rename[] options) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); bool overwrite = options != null && Arrays.AsList(options).Contains(Options.Rename .Overwrite); string error; INode srcInode = srcIIP.GetLastINode(); ValidateRenameSource(srcIIP); // validate the destination if (dst.Equals(src)) { throw new FileAlreadyExistsException("The source " + src + " and destination " + dst + " are the same"); } ValidateDestination(src, dst, srcInode); if (dstIIP.Length() == 1) { error = "rename destination cannot be the root"; NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new IOException(error); } BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite(); fsd.ezManager.CheckMoveValidity(srcIIP, dstIIP, src); INode dstInode = dstIIP.GetLastINode(); IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>(); if (dstInode != null) { // Destination exists ValidateOverwrite(src, dst, overwrite, srcInode, dstInode); FSDirSnapshotOp.CheckSnapshot(dstInode, snapshottableDirs); } INode dstParent = dstIIP.GetINode(-2); if (dstParent == null) { error = "rename destination parent " + dst + " not found."; NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new FileNotFoundException(error); } if (!dstParent.IsDirectory()) { error = "rename destination parent " + dst + " is a file."; NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error); throw new ParentNotDirectoryException(error); } // Ensure dst has quota to accommodate rename VerifyFsLimitsForRename(fsd, srcIIP, dstIIP); VerifyQuotaForRename(fsd, srcIIP, dstIIP); FSDirRenameOp.RenameOperation tx = new FSDirRenameOp.RenameOperation(fsd, src, dst , srcIIP, dstIIP); bool undoRemoveSrc = true; tx.RemoveSrc(); bool undoRemoveDst = false; long removedNum = 0; try { if (dstInode != null) { // dst exists, remove it removedNum = tx.RemoveDst(); if (removedNum != -1) { undoRemoveDst = true; } } // add src as dst to complete rename if (tx.AddSourceToDestination()) { undoRemoveSrc = false; if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedRenameTo: " + src + " is renamed to " + dst); } tx.UpdateMtimeAndLease(timestamp); // Collect the blocks and remove the lease for previous dst bool filesDeleted = false; if (undoRemoveDst) { undoRemoveDst = false; if (removedNum > 0) { filesDeleted = tx.CleanDst(bsps, collectedBlocks); } } if (snapshottableDirs.Count > 0) { // There are snapshottable directories (without snapshots) to be // deleted. Need to update the SnapshotManager. fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs); } tx.UpdateQuotasInSourceTree(bsps); return(filesDeleted); } } finally { if (undoRemoveSrc) { tx.RestoreSource(); } if (undoRemoveDst) { // Rename failed - restore dst tx.RestoreDst(bsps); } } NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename " + src + " to " + dst); throw new IOException("rename from " + src + " to " + dst + " failed."); }
/// <summary>for snapshot file.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotPathINodes() { // Create a snapshot for the dir, and check the inodes for the path // pointing to a snapshot file hdfs.AllowSnapshot(sub1); hdfs.CreateSnapshot(sub1, "s1"); // The path when accessing the snapshot file of file1 is // /TestSnapshot/sub1/.snapshot/s1/file1 string snapshotPath = sub1.ToString() + "/.snapshot/s1/file1"; string[] names = INode.GetPathNames(snapshotPath); byte[][] components = INode.GetPathComponents(names); INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore // ".snapshot" NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1); // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s1, file1} Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = GetSnapshot(nodesInPath , "s1", 3); AssertSnapshot(nodesInPath, true, snapshot, 3); // Check the INode for file1 (snapshot file) INode snapshotFileNode = nodesInPath.GetLastINode(); AssertINodeFile(snapshotFileNode, file1); NUnit.Framework.Assert.IsTrue(snapshotFileNode.GetParent().IsWithSnapshot()); // Call getExistingPathINodes and request only one INode. nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1); AssertSnapshot(nodesInPath, true, snapshot, 3); // Check the INode for file1 (snapshot file) AssertINodeFile(nodesInPath.GetLastINode(), file1); // Resolve the path "/TestSnapshot/sub1/.snapshot" string dotSnapshotPath = sub1.ToString() + "/.snapshot"; names = INode.GetPathNames(dotSnapshotPath); components = INode.GetPathComponents(names); nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false); // The number of INodes returned should still be components.length // since we put a null in the inode array for ".snapshot" NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length); // No SnapshotRoot dir is included in the resolved inodes AssertSnapshot(nodesInPath, true, snapshot, -1); // The last INode should be null, the last but 1 should be sub1 NUnit.Framework.Assert.IsNull(nodesInPath.GetLastINode()); NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(-2).GetFullPathName(), sub1. ToString()); NUnit.Framework.Assert.IsTrue(nodesInPath.GetINode(-2).IsDirectory()); string[] invalidPathComponent = new string[] { "invalidDir", "foo", ".snapshot", "bar" }; Path invalidPath = new Path(invalidPathComponent[0]); for (int i = 1; i < invalidPathComponent.Length; i++) { invalidPath = new Path(invalidPath, invalidPathComponent[i]); try { hdfs.GetFileStatus(invalidPath); NUnit.Framework.Assert.Fail(); } catch (FileNotFoundException fnfe) { System.Console.Out.WriteLine("The exception is expected: " + fnfe); } } hdfs.DeleteSnapshot(sub1, "s1"); hdfs.DisallowSnapshot(sub1); }
/// <exception cref="System.IO.IOException"/> internal static HdfsFileStatus Mkdirs(FSNamesystem fsn, string src, PermissionStatus permissions, bool createParent) { FSDirectory fsd = fsn.GetFSDirectory(); if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* NameSystem.mkdirs: " + src); } if (!DFSUtil.IsValidName(src)) { throw new InvalidPathException(src); } FSPermissionChecker pc = fsd.GetPermissionChecker(); byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src); fsd.WriteLock(); try { src = fsd.ResolvePath(pc, src, pathComponents); INodesInPath iip = fsd.GetINodesInPath4Write(src); if (fsd.IsPermissionEnabled()) { fsd.CheckTraverse(pc, iip); } INode lastINode = iip.GetLastINode(); if (lastINode != null && lastINode.IsFile()) { throw new FileAlreadyExistsException("Path is not a directory: " + src); } INodesInPath existing = lastINode != null ? iip : iip.GetExistingINodes(); if (lastINode == null) { if (fsd.IsPermissionEnabled()) { fsd.CheckAncestorAccess(pc, iip, FsAction.Write); } if (!createParent) { fsd.VerifyParentDir(iip, src); } // validate that we have enough inodes. This is, at best, a // heuristic because the mkdirs() operation might need to // create multiple inodes. fsn.CheckFsObjectLimit(); IList <string> nonExisting = iip.GetPath(existing.Length(), iip.Length() - existing .Length()); int length = nonExisting.Count; if (length > 1) { IList <string> ancestors = nonExisting.SubList(0, length - 1); // Ensure that the user can traversal the path by adding implicit // u+wx permission to all ancestor directories existing = CreateChildrenDirectories(fsd, existing, ancestors, AddImplicitUwx(permissions , permissions)); if (existing == null) { throw new IOException("Failed to create directory: " + src); } } if ((existing = CreateChildrenDirectories(fsd, existing, nonExisting.SubList(length - 1, length), permissions)) == null) { throw new IOException("Failed to create directory: " + src); } } return(fsd.GetAuditFileInfo(existing)); } finally { fsd.WriteUnlock(); } }