/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// </summary>
/// <param name="iip">the inodes resolved from the path</param>
/// <param name="collectedBlocks">blocks collected from the deleted path</param>
/// <param name="removedINodes">inodes that should be removed from inodeMap</param>
/// <param name="mtime">the time the inode is removed</param>
/// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns>
private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// check if target node exists
INode targetNode = iip.GetLastINode();
if (targetNode == null)
{
return(-1);
}
// record modification
int latestSnapshot = iip.GetLatestSnapshotId();
targetNode.RecordModification(latestSnapshot);
// Remove the node from the namespace
long removed = fsd.RemoveLastINode(iip);
if (removed == -1)
{
return(-1);
}
// set the parent's modification time
INodeDirectory parent = targetNode.GetParent();
parent.UpdateModificationTime(mtime, latestSnapshot);
fsd.UpdateCountForDelete(targetNode, iip);
if (removed == 0)
{
return(0);
}
// collect block and update quota
if (!targetNode.IsInLatestSnapshot(latestSnapshot))
{
targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks
, removedINodes);
}
else
{
QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId, latestSnapshot, collectedBlocks, removedINodes);
removed = counts.GetNameSpace();
fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation());
}
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath
() + " is removed");
}
return(removed);
}
/// <summary>
/// Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
/// dstInodes[dstInodes.length-1]
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
private static void VerifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath
dst)
{
if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
{
// Do not check quota if edits log is still being processed
return;
}
int i = 0;
while (src.GetINode(i) == dst.GetINode(i))
{
i++;
}
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
QuotaCounts delta = src.GetLastINode().ComputeQuotaUsage(bsps);
// Reduce the required quota by dst that is being removed
INode dstINode = dst.GetLastINode();
if (dstINode != null)
{
delta.Subtract(dstINode.ComputeQuotaUsage(bsps));
}
FSDirectory.VerifyQuota(dst, dst.Length() - 1, delta, src.GetINode(i - 1));
}
/// <summary>Delete a snapshot for a snapshottable directory</summary>
/// <param name="snapshotName">Name of the snapshot to be deleted</param>
/// <param name="collectedBlocks">Used to collect information to update blocksMap</param>
/// <exception cref="System.IO.IOException"/>
public virtual void DeleteSnapshot(INodesInPath iip, string snapshotName, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
INodeDirectory srcRoot = GetSnapshottableRoot(iip);
srcRoot.RemoveSnapshot(fsdir.GetBlockStoragePolicySuite(), snapshotName, collectedBlocks
, removedINodes);
numSnapshots.GetAndDecrement();
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
internal RenameOperation(FSDirectory fsd, string src, string dst, INodesInPath srcIIP
, INodesInPath dstIIP)
{
this.fsd = fsd;
this.src = src;
this.dst = dst;
this.srcIIP = srcIIP;
this.dstIIP = dstIIP;
this.srcParentIIP = srcIIP.GetParentINodesInPath();
this.dstParentIIP = dstIIP.GetParentINodesInPath();
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
srcChild = this.srcIIP.GetLastINode();
srcChildName = srcChild.GetLocalNameBytes();
int srcLatestSnapshotId = srcIIP.GetLatestSnapshotId();
isSrcInSnapshot = srcChild.IsInLatestSnapshot(srcLatestSnapshotId);
srcChildIsReference = srcChild.IsReference();
srcParent = this.srcIIP.GetINode(-2).AsDirectory();
// Record the snapshot on srcChild. After the rename, before any new
// snapshot is taken on the dst tree, changes will be recorded in the
// latest snapshot of the src tree.
if (isSrcInSnapshot)
{
srcChild.RecordModification(srcLatestSnapshotId);
}
// check srcChild for reference
srcRefDstSnapshot = srcChildIsReference ? srcChild.AsReference().GetDstSnapshotId
() : Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId;
oldSrcCounts = new QuotaCounts.Builder().Build();
if (isSrcInSnapshot)
{
INodeReference.WithName withName = srcParent.ReplaceChild4ReferenceWithName(srcChild
, srcLatestSnapshotId);
withCount = (INodeReference.WithCount)withName.GetReferredINode();
srcChild = withName;
this.srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, srcChild);
// get the counts before rename
withCount.GetReferredINode().ComputeQuotaUsage(bsps, oldSrcCounts, true);
}
else
{
if (srcChildIsReference)
{
// srcChild is reference but srcChild is not in latest snapshot
withCount = (INodeReference.WithCount)srcChild.AsReference().GetReferredINode();
}
else
{
withCount = null;
}
}
}
private static QuotaCounts ComputeQuotaDeltas(FSDirectory fsd, INodeFile target,
INodeFile[] srcList)
{
QuotaCounts deltas = new QuotaCounts.Builder().Build();
short targetRepl = target.GetBlockReplication();
foreach (INodeFile src in srcList)
{
short srcRepl = src.GetBlockReplication();
long fileSize = src.ComputeFileSize();
if (targetRepl != srcRepl)
{
deltas.AddStorageSpace(fileSize * (targetRepl - srcRepl));
BlockStoragePolicy bsp = fsd.GetBlockStoragePolicySuite().GetPolicy(src.GetStoragePolicyID
());
if (bsp != null)
{
IList <StorageType> srcTypeChosen = bsp.ChooseStorageTypes(srcRepl);
foreach (StorageType t in srcTypeChosen)
{
if (t.SupportTypeQuota())
{
deltas.AddTypeSpace(t, -fileSize);
}
}
IList <StorageType> targetTypeChosen = bsp.ChooseStorageTypes(targetRepl);
foreach (StorageType t_1 in targetTypeChosen)
{
if (t_1.SupportTypeQuota())
{
deltas.AddTypeSpace(t_1, fileSize);
}
}
}
}
}
return(deltas);
}
/// <summary>
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.ClientProtocol.SetQuota(string, long, long, Org.Apache.Hadoop.FS.StorageType)
/// "/>
/// for the contract.
/// Sets quota for for a directory.
/// </summary>
/// <returns>INodeDirectory if any of the quotas have changed. null otherwise.</returns>
/// <exception cref="System.IO.FileNotFoundException">if the path does not exist.</exception>
/// <exception cref="Org.Apache.Hadoop.FS.PathIsNotDirectoryException">if the path is not a directory.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">
/// if the directory tree size is
/// greater than the given quota
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException">if a symlink is encountered in src.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException">if path is in RO snapshot
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.UnsupportedActionException
/// "/>
internal static INodeDirectory UnprotectedSetQuota(FSDirectory fsd, string src, long
nsQuota, long ssQuota, StorageType type)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QuotaDontSet && nsQuota != HdfsConstants
.QuotaReset) || (ssQuota < 0 && ssQuota != HdfsConstants.QuotaDontSet && ssQuota
!= HdfsConstants.QuotaReset))
{
throw new ArgumentException("Illegal value for nsQuota or " + "ssQuota : " + nsQuota
+ " and " + ssQuota);
}
// sanity check for quota by storage type
if ((type != null) && (!fsd.IsQuotaByStorageTypeEnabled() || nsQuota != HdfsConstants
.QuotaDontSet))
{
throw new UnsupportedActionException("Failed to set quota by storage type because either"
+ DFSConfigKeys.DfsQuotaByStoragetypeEnabledKey + " is set to " + fsd.IsQuotaByStorageTypeEnabled
() + " or nsQuota value is illegal " + nsQuota);
}
string srcs = FSDirectory.NormalizePath(src);
INodesInPath iip = fsd.GetINodesInPath4Write(srcs, true);
INodeDirectory dirNode = INodeDirectory.ValueOf(iip.GetLastINode(), srcs);
if (dirNode.IsRoot() && nsQuota == HdfsConstants.QuotaReset)
{
throw new ArgumentException("Cannot clear namespace quota on root.");
}
else
{
// a directory inode
QuotaCounts oldQuota = dirNode.GetQuotaCounts();
long oldNsQuota = oldQuota.GetNameSpace();
long oldSsQuota = oldQuota.GetStorageSpace();
if (nsQuota == HdfsConstants.QuotaDontSet)
{
nsQuota = oldNsQuota;
}
if (ssQuota == HdfsConstants.QuotaDontSet)
{
ssQuota = oldSsQuota;
}
// unchanged space/namespace quota
if (type == null && oldNsQuota == nsQuota && oldSsQuota == ssQuota)
{
return(null);
}
// unchanged type quota
if (type != null)
{
EnumCounters <StorageType> oldTypeQuotas = oldQuota.GetTypeSpaces();
if (oldTypeQuotas != null && oldTypeQuotas.Get(type) == ssQuota)
{
return(null);
}
}
int latest = iip.GetLatestSnapshotId();
dirNode.RecordModification(latest);
dirNode.SetQuota(fsd.GetBlockStoragePolicySuite(), nsQuota, ssQuota, type);
return(dirNode);
}
}
/// <summary>Rename src to dst.</summary>
/// <remarks>
/// Rename src to dst.
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.DistributedFileSystem.Rename(Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Options.Rename[])
/// "/>
/// for details related to rename semantics and exceptions.
/// </remarks>
/// <param name="fsd">FSDirectory</param>
/// <param name="src">source path</param>
/// <param name="dst">destination path</param>
/// <param name="timestamp">modification time</param>
/// <param name="collectedBlocks">blocks to be removed</param>
/// <param name="options">Rename options</param>
/// <returns>whether a file/directory gets overwritten in the dst path</returns>
/// <exception cref="System.IO.IOException"/>
internal static bool UnprotectedRenameTo(FSDirectory fsd, string src, string dst,
INodesInPath srcIIP, INodesInPath dstIIP, long timestamp, INode.BlocksMapUpdateInfo
collectedBlocks, params Options.Rename[] options)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
bool overwrite = options != null && Arrays.AsList(options).Contains(Options.Rename
.Overwrite);
string error;
INode srcInode = srcIIP.GetLastINode();
ValidateRenameSource(srcIIP);
// validate the destination
if (dst.Equals(src))
{
throw new FileAlreadyExistsException("The source " + src + " and destination " +
dst + " are the same");
}
ValidateDestination(src, dst, srcInode);
if (dstIIP.Length() == 1)
{
error = "rename destination cannot be the root";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new IOException(error);
}
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
fsd.ezManager.CheckMoveValidity(srcIIP, dstIIP, src);
INode dstInode = dstIIP.GetLastINode();
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
if (dstInode != null)
{
// Destination exists
ValidateOverwrite(src, dst, overwrite, srcInode, dstInode);
FSDirSnapshotOp.CheckSnapshot(dstInode, snapshottableDirs);
}
INode dstParent = dstIIP.GetINode(-2);
if (dstParent == null)
{
error = "rename destination parent " + dst + " not found.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new FileNotFoundException(error);
}
if (!dstParent.IsDirectory())
{
error = "rename destination parent " + dst + " is a file.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new ParentNotDirectoryException(error);
}
// Ensure dst has quota to accommodate rename
VerifyFsLimitsForRename(fsd, srcIIP, dstIIP);
VerifyQuotaForRename(fsd, srcIIP, dstIIP);
FSDirRenameOp.RenameOperation tx = new FSDirRenameOp.RenameOperation(fsd, src, dst
, srcIIP, dstIIP);
bool undoRemoveSrc = true;
tx.RemoveSrc();
bool undoRemoveDst = false;
long removedNum = 0;
try
{
if (dstInode != null)
{
// dst exists, remove it
removedNum = tx.RemoveDst();
if (removedNum != -1)
{
undoRemoveDst = true;
}
}
// add src as dst to complete rename
if (tx.AddSourceToDestination())
{
undoRemoveSrc = false;
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedRenameTo: " + src + " is renamed to "
+ dst);
}
tx.UpdateMtimeAndLease(timestamp);
// Collect the blocks and remove the lease for previous dst
bool filesDeleted = false;
if (undoRemoveDst)
{
undoRemoveDst = false;
if (removedNum > 0)
{
filesDeleted = tx.CleanDst(bsps, collectedBlocks);
}
}
if (snapshottableDirs.Count > 0)
{
// There are snapshottable directories (without snapshots) to be
// deleted. Need to update the SnapshotManager.
fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs);
}
tx.UpdateQuotasInSourceTree(bsps);
return(filesDeleted);
}
}
finally
{
if (undoRemoveSrc)
{
tx.RestoreSource();
}
if (undoRemoveDst)
{
// Rename failed - restore dst
tx.RestoreDst(bsps);
}
}
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst);
throw new IOException("rename from " + src + " to " + dst + " failed.");
}
internal static bool UnprotectedRenameTo(FSDirectory fsd, string src, string dst,
INodesInPath srcIIP, INodesInPath dstIIP, long timestamp)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INode srcInode = srcIIP.GetLastINode();
try
{
ValidateRenameSource(srcIIP);
}
catch (SnapshotException e)
{
throw;
}
catch (IOException)
{
return(false);
}
// validate the destination
if (dst.Equals(src))
{
return(true);
}
try
{
ValidateDestination(src, dst, srcInode);
}
catch (IOException)
{
return(false);
}
if (dstIIP.GetLastINode() != null)
{
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst + " because destination " + "exists");
return(false);
}
INode dstParent = dstIIP.GetINode(-2);
if (dstParent == null)
{
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst + " because destination's " + "parent does not exist");
return(false);
}
fsd.ezManager.CheckMoveValidity(srcIIP, dstIIP, src);
// Ensure dst has quota to accommodate rename
VerifyFsLimitsForRename(fsd, srcIIP, dstIIP);
VerifyQuotaForRename(fsd, srcIIP, dstIIP);
FSDirRenameOp.RenameOperation tx = new FSDirRenameOp.RenameOperation(fsd, src, dst
, srcIIP, dstIIP);
bool added = false;
try
{
// remove src
if (!tx.RemoveSrc4OldRename())
{
return(false);
}
added = tx.AddSourceToDestination();
if (added)
{
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory" + ".unprotectedRenameTo: " + src
+ " is renamed to " + dst);
}
tx.UpdateMtimeAndLease(timestamp);
tx.UpdateQuotasInSourceTree(fsd.GetBlockStoragePolicySuite());
return(true);
}
}
finally
{
if (!added)
{
tx.RestoreSource();
}
}
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst);
return(false);
}