/// <summary>Delete a snapshot of a snapshottable directory</summary>
/// <param name="snapshotRoot">The snapshottable directory</param>
/// <param name="snapshotName">The name of the to-be-deleted snapshot</param>
/// <exception cref="System.IO.IOException"/>
internal static INode.BlocksMapUpdateInfo DeleteSnapshot(FSDirectory fsd, SnapshotManager
snapshotManager, string snapshotRoot, string snapshotName, bool logRetryCache)
{
INodesInPath iip = fsd.GetINodesInPath4Write(snapshotRoot);
if (fsd.IsPermissionEnabled())
{
FSPermissionChecker pc = fsd.GetPermissionChecker();
fsd.CheckOwner(pc, iip);
}
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
ChunkedArrayList <INode> removedINodes = new ChunkedArrayList <INode>();
fsd.WriteLock();
try
{
snapshotManager.DeleteSnapshot(iip, snapshotName, collectedBlocks, removedINodes);
fsd.RemoveFromInodeMap(removedINodes);
}
finally
{
fsd.WriteUnlock();
}
removedINodes.Clear();
fsd.GetEditLog().LogDeleteSnapshot(snapshotRoot, snapshotName, logRetryCache);
return(collectedBlocks);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/>
public virtual Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot RemoveSnapshot
(BlockStoragePolicySuite bsps, string snapshotName, INode.BlocksMapUpdateInfo collectedBlocks
, IList <INode> removedINodes)
{
return(GetDirectorySnapshottableFeature().RemoveSnapshot(bsps, this, snapshotName
, collectedBlocks, removedINodes));
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to
// the previous WithName instance
if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
prior = GetPriorSnapshot(this);
}
// if prior is not NO_SNAPSHOT_ID, and prior is not before the
// to-be-deleted snapshot, we can quit here and leave the snapshot
// deletion work to the src tree of rename
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
.Compare(snapshot, prior) <= 0)
{
return(new QuotaCounts.Builder().Build());
}
return(GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes
));
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
return(sf.CleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, removedINodes
));
}
QuotaCounts counts = new QuotaCounts.Builder().Build();
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
// this only happens when deleting the current file and the file is not
// in any snapshot
ComputeQuotaUsage(bsps, counts, false);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// when deleting the current file and the file is in snapshot, we should
// clean the 0-sized block if the file is UC
FileUnderConstructionFeature uc = GetFileUnderConstructionFeature();
if (uc != null)
{
uc.CleanZeroSizeBlock(this, collectedBlocks);
}
}
}
return(counts);
}
/// <summary>Exclude blocks collected for deletion that belong to a snapshot.</summary>
internal virtual void ExcludeSnapshotBlocks(int snapshotId, INode.BlocksMapUpdateInfo
collectedBlocks)
{
if (collectedBlocks == null || collectedBlocks.GetToDeleteList().IsEmpty())
{
return;
}
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf == null)
{
return;
}
BlockInfoContiguous[] snapshotBlocks = GetDiffs().FindEarlierSnapshotBlocks(snapshotId
);
if (snapshotBlocks == null)
{
return;
}
IList <Block> toDelete = collectedBlocks.GetToDeleteList();
foreach (Block blk in snapshotBlocks)
{
if (toDelete.Contains(blk))
{
collectedBlocks.RemoveDeleteBlock(blk);
}
}
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (blocks != null && collectedBlocks != null)
{
foreach (BlockInfoContiguous blk in blocks)
{
collectedBlocks.AddDeleteBlock(blk);
blk.SetBlockCollection(null);
}
}
SetBlocks(null);
if (GetAclFeature() != null)
{
AclStorage.RemoveAclFeature(GetAclFeature());
}
Clear();
removedINodes.AddItem(this);
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
sf.GetDiffs().DestroyAndCollectSnapshotBlocks(collectedBlocks);
sf.ClearDiffs();
}
}
/// <summary>Remove full blocks at the end file up to newLength</summary>
/// <returns>sum of sizes of the remained blocks</returns>
public virtual long CollectBlocksBeyondMax(long max, INode.BlocksMapUpdateInfo collectedBlocks
)
{
BlockInfoContiguous[] oldBlocks = GetBlocks();
if (oldBlocks == null)
{
return(0);
}
// find the minimum n such that the size of the first n blocks > max
int n = 0;
long size = 0;
for (; n < oldBlocks.Length && max > size; n++)
{
size += oldBlocks[n].GetNumBytes();
}
if (n >= oldBlocks.Length)
{
return(size);
}
// starting from block n, the data is beyond max.
// resize the array.
TruncateBlocksTo(n);
// collect the blocks beyond max
if (collectedBlocks != null)
{
for (; n < oldBlocks.Length; n++)
{
collectedBlocks.AddDeleteBlock(oldBlocks[n]);
}
}
return(size);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// there is snapshot data
if (sf != null)
{
return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes));
}
// there is no snapshot data
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
// destroy the whole subtree and collect blocks that should be deleted
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// process recursively down the subtree
QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes, null);
if (IsQuotaSet())
{
GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation());
}
return(counts);
}
}
/// <summary>Delete the target directory and collect the blocks under it</summary>
/// <param name="iip">the INodesInPath instance containing all the INodes for the path
/// </param>
/// <param name="collectedBlocks">Blocks under the deleted directory</param>
/// <param name="removedINodes">INodes that should be removed from inodeMap</param>
/// <returns>the number of files that have been removed</returns>
/// <exception cref="System.IO.IOException"/>
internal static long Delete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes, long mtime)
{
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.delete: " + iip.GetPath());
}
long filesRemoved;
fsd.WriteLock();
try
{
if (!DeleteAllowed(iip, iip.GetPath()))
{
filesRemoved = -1;
}
else
{
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs);
filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime);
fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs);
}
}
finally
{
fsd.WriteUnlock();
}
return(filesRemoved);
}
/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// <br />
/// Note: This is to be used by
/// <see cref="FSEditLog"/>
/// only.
/// <br />
/// </summary>
/// <param name="src">a string representation of a path to an inode</param>
/// <param name="mtime">the time the inode is removed</param>
/// <exception cref="System.IO.IOException"/>
internal static void DeleteForEditLog(FSDirectory fsd, string src, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
FSNamesystem fsn = fsd.GetFSNamesystem();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
IList <INode> removedINodes = new ChunkedArrayList <INode>();
INodesInPath iip = fsd.GetINodesInPath4Write(FSDirectory.NormalizePath(src), false
);
if (!DeleteAllowed(iip, src))
{
return;
}
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs);
long filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime
);
fsn.RemoveSnapshottableDirs(snapshottableDirs);
if (filesRemoved >= 0)
{
fsn.RemoveLeasesAndINodes(src, removedINodes, false);
fsn.RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
}
/// <summary>The new rename which has the POSIX semantic.</summary>
/// <exception cref="System.IO.IOException"/>
internal static KeyValuePair <INode.BlocksMapUpdateInfo, HdfsFileStatus> RenameToInt
(FSDirectory fsd, string srcArg, string dstArg, bool logRetryCache, params Options.Rename
[] options)
{
string src = srcArg;
string dst = dstArg;
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* NameSystem.renameTo: with options -" + " " +
src + " to " + dst);
}
if (!DFSUtil.IsValidName(dst))
{
throw new InvalidPathException("Invalid name: " + dst);
}
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] srcComponents = FSDirectory.GetPathComponentsForReservedPath(src);
byte[][] dstComponents = FSDirectory.GetPathComponentsForReservedPath(dst);
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
src = fsd.ResolvePath(pc, src, srcComponents);
dst = fsd.ResolvePath(pc, dst, dstComponents);
RenameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
INodesInPath dstIIP = fsd.GetINodesInPath(dst, false);
HdfsFileStatus resultingStat = fsd.GetAuditFileInfo(dstIIP);
return(new AbstractMap.SimpleImmutableEntry <INode.BlocksMapUpdateInfo, HdfsFileStatus
>(collectedBlocks, resultingStat));
}
/// <summary>Call cleanSubtree(..) recursively down the subtree.</summary>
public virtual QuotaCounts CleanSubtreeRecursively(BlockStoragePolicySuite bsps,
int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode>
removedINodes, IDictionary <INode, INode> excludedNodes)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
// in case of deletion snapshot, since this call happens after we modify
// the diff list, the snapshot to be deleted has been combined or renamed
// to its latest previous snapshot. (besides, we also need to consider nodes
// created after prior but before snapshot. this will be done in
// DirectoryWithSnapshotFeature)
int s = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId
? prior : snapshot;
foreach (INode child in GetChildrenList(s))
{
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
excludedNodes != null && excludedNodes.Contains(child))
{
continue;
}
else
{
QuotaCounts childCounts = child.CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
counts.Add(childCounts);
}
}
return(counts);
}
/// <summary>Remove a file/directory from the namespace.</summary>
/// <remarks>
/// Remove a file/directory from the namespace.
/// <p>
/// For large directories, deletion is incremental. The blocks under
/// the directory are collected and deleted a small number at a time holding
/// the
/// <see cref="FSNamesystem"/>
/// lock.
/// <p>
/// For small directory or file the deletion is done in one shot.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static INode.BlocksMapUpdateInfo DeleteInternal(FSNamesystem fsn, string
src, INodesInPath iip, bool logRetryCache)
{
System.Diagnostics.Debug.Assert(fsn.HasWriteLock());
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* NameSystem.delete: " + src);
}
FSDirectory fsd = fsn.GetFSDirectory();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
IList <INode> removedINodes = new ChunkedArrayList <INode>();
long mtime = Time.Now();
// Unlink the target directory from directory tree
long filesRemoved = Delete(fsd, iip, collectedBlocks, removedINodes, mtime);
if (filesRemoved < 0)
{
return(null);
}
fsd.GetEditLog().LogDelete(src, mtime, logRetryCache);
IncrDeletedFileCount(filesRemoved);
fsn.RemoveLeasesAndINodes(src, removedINodes, true);
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* Namesystem.delete: " + src + " is removed");
}
return(collectedBlocks);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
// used by WithCount
return(referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes
));
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
// used by WithCount
if (RemoveReference(this) <= 0)
{
referred.DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
if (snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.
NoSnapshotId)
{
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
return(new QuotaCounts.Builder().NameSpace(1).Build());
}
/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// </summary>
/// <param name="iip">the inodes resolved from the path</param>
/// <param name="collectedBlocks">blocks collected from the deleted path</param>
/// <param name="removedINodes">inodes that should be removed from inodeMap</param>
/// <param name="mtime">the time the inode is removed</param>
/// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns>
private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// check if target node exists
INode targetNode = iip.GetLastINode();
if (targetNode == null)
{
return(-1);
}
// record modification
int latestSnapshot = iip.GetLatestSnapshotId();
targetNode.RecordModification(latestSnapshot);
// Remove the node from the namespace
long removed = fsd.RemoveLastINode(iip);
if (removed == -1)
{
return(-1);
}
// set the parent's modification time
INodeDirectory parent = targetNode.GetParent();
parent.UpdateModificationTime(mtime, latestSnapshot);
fsd.UpdateCountForDelete(targetNode, iip);
if (removed == 0)
{
return(0);
}
// collect block and update quota
if (!targetNode.IsInLatestSnapshot(latestSnapshot))
{
targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks
, removedINodes);
}
else
{
QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId, latestSnapshot, collectedBlocks, removedINodes);
removed = counts.GetNameSpace();
fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation());
}
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath
() + " is removed");
}
return(removed);
}
/// <summary>Rename src to dst.</summary>
/// <remarks>
/// Rename src to dst.
/// <br />
/// Note: This is to be used by
/// <see cref="FSEditLogLoader"/>
/// only.
/// <br />
/// </remarks>
/// <param name="fsd">FSDirectory</param>
/// <param name="src">source path</param>
/// <param name="dst">destination path</param>
/// <param name="timestamp">modification time</param>
/// <param name="options">Rename options</param>
/// <exception cref="System.IO.IOException"/>
internal static bool RenameForEditLog(FSDirectory fsd, string src, string dst, long
timestamp, params Options.Rename[] options)
{
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
INodesInPath srcIIP = fsd.GetINodesInPath4Write(src, false);
INodesInPath dstIIP = fsd.GetINodesInPath4Write(dst, false);
bool ret = UnprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp, collectedBlocks
, options);
if (!collectedBlocks.GetToDeleteList().IsEmpty())
{
fsd.GetFSNamesystem().RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
return(ret);
}
/// <summary>
/// When deleting a file in the current fs directory, and the file is contained
/// in a snapshot, we should delete the last block if it's under construction
/// and its size is 0.
/// </summary>
internal virtual void CleanZeroSizeBlock(INodeFile f, INode.BlocksMapUpdateInfo collectedBlocks
)
{
BlockInfoContiguous[] blocks = f.GetBlocks();
if (blocks != null && blocks.Length > 0 && blocks[blocks.Length - 1] is BlockInfoContiguousUnderConstruction)
{
BlockInfoContiguousUnderConstruction lastUC = (BlockInfoContiguousUnderConstruction
)blocks[blocks.Length - 1];
if (lastUC.GetNumBytes() == 0)
{
// this is a 0-sized block. do not need check its UC state here
collectedBlocks.AddDeleteBlock(lastUC);
f.RemoveLastBlock(lastUC);
}
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
// since WithName node resides in deleted list acting as a snapshot copy,
// the parameter snapshot must be non-null
Preconditions.CheckArgument(snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
// previous WithName instance
if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
prior = GetPriorSnapshot(this);
}
if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
.Compare(snapshot, prior) <= 0)
{
return(new QuotaCounts.Builder().Build());
}
QuotaCounts counts = GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
INodeReference @ref = GetReferredINode().GetParentReference();
if (@ref != null)
{
try
{
@ref.AddSpaceConsumed(counts.Negation(), true);
}
catch (QuotaExceededException)
{
Org.Mortbay.Log.Log.Warn("Should not have QuotaExceededException");
}
}
if (snapshot < lastSnapshotId)
{
// for a WithName node, when we compute its quota usage, we only count
// in all the nodes existing at the time of the corresponding rename op.
// Thus if we are deleting a snapshot before/at the snapshot associated
// with lastSnapshotId, we do not need to update the quota upwards.
counts = new QuotaCounts.Builder().Build();
}
return(counts);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
internal virtual bool CleanDst(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks)
{
Preconditions.CheckState(oldDstChild != null);
IList <INode> removedINodes = new ChunkedArrayList <INode>();
bool filesDeleted;
if (!oldDstChild.IsInLatestSnapshot(dstIIP.GetLatestSnapshotId()))
{
oldDstChild.DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
filesDeleted = true;
}
else
{
filesDeleted = oldDstChild.CleanSubtree(bsps, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId, dstIIP.GetLatestSnapshotId(), collectedBlocks, removedINodes).GetNameSpace
() >= 0;
}
fsd.GetFSNamesystem().RemoveLeasesAndINodes(src, removedINodes, false);
return(filesDeleted);
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf != null)
{
sf.Clear(bsps, this, collectedBlocks, removedINodes);
}
foreach (INode child in GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId))
{
child.DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
if (GetAclFeature() != null)
{
AclStorage.RemoveAclFeature(GetAclFeature());
}
Clear();
removedINodes.AddItem(this);
}
public virtual void CollectBlocksBeyondSnapshot(BlockInfoContiguous[] snapshotBlocks
, INode.BlocksMapUpdateInfo collectedBlocks)
{
BlockInfoContiguous[] oldBlocks = GetBlocks();
if (snapshotBlocks == null || oldBlocks == null)
{
return;
}
// Skip blocks in common between the file and the snapshot
int n = 0;
while (n < oldBlocks.Length && n < snapshotBlocks.Length && oldBlocks[n] == snapshotBlocks
[n])
{
n++;
}
TruncateBlocksTo(n);
// Collect the remaining blocks of the file
while (n < oldBlocks.Length)
{
collectedBlocks.AddDeleteBlock(oldBlocks[n++]);
}
}
/// <seealso>
///
/// <see cref="UnprotectedRenameTo(FSDirectory, string, string, INodesInPath, INodesInPath, long, BlocksMapUpdateInfo, Org.Apache.Hadoop.FS.Options.Rename[])
/// "/>
/// </seealso>
/// <exception cref="System.IO.IOException"/>
internal static void RenameTo(FSDirectory fsd, FSPermissionChecker pc, string src
, string dst, INode.BlocksMapUpdateInfo collectedBlocks, bool logRetryCache, params
Options.Rename[] options)
{
INodesInPath srcIIP = fsd.GetINodesInPath4Write(src, false);
INodesInPath dstIIP = fsd.GetINodesInPath4Write(dst, false);
if (fsd.IsPermissionEnabled())
{
// Rename does not operate on link targets
// Do not resolveLink when checking permissions of src and dst
// Check write access to parent of src
fsd.CheckPermission(pc, srcIIP, false, null, FsAction.Write, null, null, false);
// Check write access to ancestor of dst
fsd.CheckPermission(pc, dstIIP, false, FsAction.Write, null, null, null, false);
}
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.renameTo: " + src + " to " + dst);
}
long mtime = Time.Now();
fsd.WriteLock();
try
{
if (UnprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, mtime, collectedBlocks, options
))
{
FSDirDeleteOp.IncrDeletedFileCount(1);
}
}
finally
{
fsd.WriteUnlock();
}
fsd.GetEditLog().LogRename(src, dst, mtime, logRetryCache, options);
}
/// <summary>
/// Destroy self and clear everything! If the INode is a file, this method
/// collects its blocks for further block deletion.
/// </summary>
/// <remarks>
/// Destroy self and clear everything! If the INode is a file, this method
/// collects its blocks for further block deletion. If the INode is a
/// directory, the method goes down the subtree and collects blocks from the
/// descents, and clears its parent/children references as well. The method
/// also clears the diff list if the INode contains snapshot diff list.
/// </remarks>
/// <param name="bsps">
/// block storage policy suite to calculate intended storage type usage
/// This is needed because INodeReference#destroyAndCollectBlocks() needs
/// to call INode#cleanSubtree(), which calls INode#computeQuotaUsage().
/// </param>
/// <param name="collectedBlocks">
/// blocks collected from the descents for further block
/// deletion/update will be added to this map.
/// </param>
/// <param name="removedINodes">
/// INodes collected from the descents for further cleaning up of
/// inodeMap
/// </param>
public abstract void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <Org.Apache.Hadoop.Hdfs.Server.Namenode.INode> removedINodes
);
/// <summary>
/// Clean the subtree under this inode and collect the blocks from the descents
/// for further block deletion/update.
/// </summary>
/// <remarks>
/// Clean the subtree under this inode and collect the blocks from the descents
/// for further block deletion/update. The current inode can either resides in
/// the current tree or be stored as a snapshot copy.
/// <pre>
/// In general, we have the following rules.
/// 1. When deleting a file/directory in the current tree, we have different
/// actions according to the type of the node to delete.
/// 1.1 The current inode (this) is an
/// <see cref="INodeFile"/>
/// .
/// 1.1.1 If
/// <c>prior</c>
/// is null, there is no snapshot taken on ancestors
/// before. Thus we simply destroy (i.e., to delete completely, no need to save
/// snapshot copy) the current INode and collect its blocks for further
/// cleansing.
/// 1.1.2 Else do nothing since the current INode will be stored as a snapshot
/// copy.
/// 1.2 The current inode is an
/// <see cref="INodeDirectory"/>
/// .
/// 1.2.1 If
/// <c>prior</c>
/// is null, there is no snapshot taken on ancestors
/// before. Similarly, we destroy the whole subtree and collect blocks.
/// 1.2.2 Else do nothing with the current INode. Recursively clean its
/// children.
/// 1.3 The current inode is a file with snapshot.
/// Call recordModification(..) to capture the current states.
/// Mark the INode as deleted.
/// 1.4 The current inode is an
/// <see cref="INodeDirectory"/>
/// with snapshot feature.
/// Call recordModification(..) to capture the current states.
/// Destroy files/directories created after the latest snapshot
/// (i.e., the inodes stored in the created list of the latest snapshot).
/// Recursively clean remaining children.
/// 2. When deleting a snapshot.
/// 2.1 To clean
/// <see cref="INodeFile"/>
/// : do nothing.
/// 2.2 To clean
/// <see cref="INodeDirectory"/>
/// : recursively clean its children.
/// 2.3 To clean INodeFile with snapshot: delete the corresponding snapshot in
/// its diff list.
/// 2.4 To clean
/// <see cref="INodeDirectory"/>
/// with snapshot: delete the corresponding
/// snapshot in its diff list. Recursively clean its children.
/// </pre>
/// </remarks>
/// <param name="bsps">block storage policy suite to calculate intended storage type usage
/// </param>
/// <param name="snapshotId">
/// The id of the snapshot to delete.
/// <see cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId
/// "/>
/// means to delete the current
/// file/directory.
/// </param>
/// <param name="priorSnapshotId">
/// The id of the latest snapshot before the to-be-deleted snapshot.
/// When deleting a current inode, this parameter captures the latest
/// snapshot.
/// </param>
/// <param name="collectedBlocks">
/// blocks collected from the descents for further block
/// deletion/update will be added to the given map.
/// </param>
/// <param name="removedINodes">
/// INodes collected from the descents for further cleaning up of
/// inodeMap
/// </param>
/// <returns>quota usage delta when deleting a snapshot</returns>
public abstract QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <Org.Apache.Hadoop.Hdfs.Server.Namenode.INode
> removedINodes);
/// <summary>Rename src to dst.</summary>
/// <remarks>
/// Rename src to dst.
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.DistributedFileSystem.Rename(Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Options.Rename[])
/// "/>
/// for details related to rename semantics and exceptions.
/// </remarks>
/// <param name="fsd">FSDirectory</param>
/// <param name="src">source path</param>
/// <param name="dst">destination path</param>
/// <param name="timestamp">modification time</param>
/// <param name="collectedBlocks">blocks to be removed</param>
/// <param name="options">Rename options</param>
/// <returns>whether a file/directory gets overwritten in the dst path</returns>
/// <exception cref="System.IO.IOException"/>
internal static bool UnprotectedRenameTo(FSDirectory fsd, string src, string dst,
INodesInPath srcIIP, INodesInPath dstIIP, long timestamp, INode.BlocksMapUpdateInfo
collectedBlocks, params Options.Rename[] options)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
bool overwrite = options != null && Arrays.AsList(options).Contains(Options.Rename
.Overwrite);
string error;
INode srcInode = srcIIP.GetLastINode();
ValidateRenameSource(srcIIP);
// validate the destination
if (dst.Equals(src))
{
throw new FileAlreadyExistsException("The source " + src + " and destination " +
dst + " are the same");
}
ValidateDestination(src, dst, srcInode);
if (dstIIP.Length() == 1)
{
error = "rename destination cannot be the root";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new IOException(error);
}
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
fsd.ezManager.CheckMoveValidity(srcIIP, dstIIP, src);
INode dstInode = dstIIP.GetLastINode();
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
if (dstInode != null)
{
// Destination exists
ValidateOverwrite(src, dst, overwrite, srcInode, dstInode);
FSDirSnapshotOp.CheckSnapshot(dstInode, snapshottableDirs);
}
INode dstParent = dstIIP.GetINode(-2);
if (dstParent == null)
{
error = "rename destination parent " + dst + " not found.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new FileNotFoundException(error);
}
if (!dstParent.IsDirectory())
{
error = "rename destination parent " + dst + " is a file.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new ParentNotDirectoryException(error);
}
// Ensure dst has quota to accommodate rename
VerifyFsLimitsForRename(fsd, srcIIP, dstIIP);
VerifyQuotaForRename(fsd, srcIIP, dstIIP);
FSDirRenameOp.RenameOperation tx = new FSDirRenameOp.RenameOperation(fsd, src, dst
, srcIIP, dstIIP);
bool undoRemoveSrc = true;
tx.RemoveSrc();
bool undoRemoveDst = false;
long removedNum = 0;
try
{
if (dstInode != null)
{
// dst exists, remove it
removedNum = tx.RemoveDst();
if (removedNum != -1)
{
undoRemoveDst = true;
}
}
// add src as dst to complete rename
if (tx.AddSourceToDestination())
{
undoRemoveSrc = false;
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedRenameTo: " + src + " is renamed to "
+ dst);
}
tx.UpdateMtimeAndLease(timestamp);
// Collect the blocks and remove the lease for previous dst
bool filesDeleted = false;
if (undoRemoveDst)
{
undoRemoveDst = false;
if (removedNum > 0)
{
filesDeleted = tx.CleanDst(bsps, collectedBlocks);
}
}
if (snapshottableDirs.Count > 0)
{
// There are snapshottable directories (without snapshots) to be
// deleted. Need to update the SnapshotManager.
fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs);
}
tx.UpdateQuotasInSourceTree(bsps);
return(filesDeleted);
}
}
finally
{
if (undoRemoveSrc)
{
tx.RestoreSource();
}
if (undoRemoveDst)
{
// Rename failed - restore dst
tx.RestoreDst(bsps);
}
}
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst);
throw new IOException("rename from " + src + " to " + dst + " failed.");
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
int snapshot = GetSelfSnapshot();
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
int prior = GetPriorSnapshot(this);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
snapshot <= prior)
{
// the snapshot to be deleted has been deleted while traversing
// the src tree of the previous rename operation. This usually
// happens when rename's src and dst are under the same
// snapshottable directory. E.g., the following operation sequence:
// 1. create snapshot s1 on /test
// 2. rename /test/foo/bar to /test/foo2/bar
// 3. create snapshot s2 on /test
// 4. rename foo2 again
// 5. delete snapshot s2
return;
}
try
{
QuotaCounts counts = referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
INodeReference @ref = GetReferredINode().GetParentReference();
if (@ref != null)
{
@ref.AddSpaceConsumed(counts.Negation(), true);
}
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
/// <summary>
/// <inheritDoc/>
/// <br/>
/// To destroy a DstReference node, we first remove its link with the
/// referred node. If the reference number of the referred node is <= 0, we
/// destroy the subtree of the referred node. Otherwise, we clean the
/// referred node's subtree and delete everything created after the last
/// rename operation, i.e., everything outside of the scope of the prior
/// WithName nodes.
/// </summary>
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot
int prior = GetPriorSnapshot(this);
// prior must be non-null, otherwise we do not have any previous
// WithName nodes, and the reference number will be 0.
Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId);
// identify the snapshot created after prior
int snapshot = GetSelfSnapshot(prior);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (referred.IsFile())
{
// if referred is a file, it must be a file with snapshot since we did
// recordModification before the rename
INodeFile file = referred.AsFile();
Preconditions.CheckState(file.IsWithSnapshot());
// make sure we mark the file as deleted
file.GetFileWithSnapshotFeature().DeleteCurrentFile();
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
}
else
{
if (referred.IsDirectory())
{
// similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot
INodeDirectory dir = referred.AsDirectory();
Preconditions.CheckState(dir.IsWithSnapshot());
try
{
DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks
, removedINodes);
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
removedINodes.AddItem(this);
}
