public override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps, byte
blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// we are computing the quota usage for a specific snapshot here, i.e., the
// computation only includes files/directories that exist at the time of the
// given snapshot
if (sf != null && lastSnapshotId != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId && !(useCache && IsQuotaSet()))
{
ReadOnlyList <INode> childrenList = GetChildrenList(lastSnapshotId);
foreach (INode child in childrenList)
{
byte childPolicyId = child.GetStoragePolicyIDForQuota(blockStoragePolicyId);
child.ComputeQuotaUsage(bsps, childPolicyId, counts, useCache, lastSnapshotId);
}
counts.AddNameSpace(1);
return(counts);
}
// compute the quota usage in the scope of the current directory tree
DirectoryWithQuotaFeature q = GetDirectoryWithQuotaFeature();
if (useCache && q != null && q.IsQuotaSet())
{
// use the cached quota
return(q.AddCurrentSpaceUsage(counts));
}
else
{
useCache = q != null && !q.IsQuotaSet() ? false : useCache;
return(ComputeDirectoryQuotaUsage(bsps, blockStoragePolicyId, counts, useCache, lastSnapshotId
));
}
}
private int GetSelfSnapshot(int prior)
{
INodeReference.WithCount wc = (INodeReference.WithCount)GetReferredINode().AsReference
();
INode referred = wc.GetReferredINode();
int lastSnapshot = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId;
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
lastSnapshot = referred.AsFile().GetDiffs().GetLastSnapshotId();
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
lastSnapshot = sf.GetLastSnapshotId();
}
}
}
if (lastSnapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
lastSnapshot != prior)
{
return(lastSnapshot);
}
else
{
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId);
}
}
/// <summary>Replace the given child with a new child.</summary>
/// <remarks>
/// Replace the given child with a new child. Note that we no longer need to
/// replace an normal INodeDirectory or INodeFile into an
/// INodeDirectoryWithSnapshot or INodeFileUnderConstruction. The only cases
/// for child replacement is for reference nodes.
/// </remarks>
public virtual void ReplaceChild(INode oldChild, INode newChild, INodeMap inodeMap
)
{
Preconditions.CheckNotNull(children);
int i = SearchChildren(newChild.GetLocalNameBytes());
Preconditions.CheckState(i >= 0);
Preconditions.CheckState(oldChild == children[i] || oldChild == children[i].AsReference
().GetReferredINode().AsReference().GetReferredINode());
oldChild = children[i];
if (oldChild.IsReference() && newChild.IsReference())
{
// both are reference nodes, e.g., DstReference -> WithName
INodeReference.WithCount withCount = (INodeReference.WithCount)oldChild.AsReference
().GetReferredINode();
withCount.RemoveReference(oldChild.AsReference());
}
children.Set(i, newChild);
// replace the instance in the created list of the diff list
DirectoryWithSnapshotFeature sf = this.GetDirectoryWithSnapshotFeature();
if (sf != null)
{
sf.GetDiffs().ReplaceChild(Diff.ListType.Created, oldChild, newChild);
}
// update the inodeMap
if (inodeMap != null)
{
inodeMap.Put(newChild);
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// there is snapshot data
if (sf != null)
{
return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes));
}
// there is no snapshot data
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
// destroy the whole subtree and collect blocks that should be deleted
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// process recursively down the subtree
QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes, null);
if (IsQuotaSet())
{
GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation());
}
return(counts);
}
}
/// <summary>Add a child inode to the directory.</summary>
/// <param name="node">INode to insert</param>
/// <param name="setModTime">
/// set modification time for the parent node
/// not needed when replaying the addition and
/// the parent already has the proper mod time
/// </param>
/// <returns>
/// false if the child with this name already exists;
/// otherwise, return true;
/// </returns>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
public virtual bool AddChild(INode node, bool setModTime, int latestSnapshotId)
{
int low = SearchChildren(node.GetLocalNameBytes());
if (low >= 0)
{
return(false);
}
if (IsInLatestSnapshot(latestSnapshotId))
{
// create snapshot feature if necessary
DirectoryWithSnapshotFeature sf = this.GetDirectoryWithSnapshotFeature();
if (sf == null)
{
sf = this.AddSnapshotFeature(null);
}
return(sf.AddChild(this, node, setModTime, latestSnapshotId));
}
AddChild(node, low);
if (setModTime)
{
// update modification time of the parent directory
UpdateModificationTime(node.GetModificationTime(), latestSnapshotId);
}
return(true);
}
public virtual DirectoryWithSnapshotFeature AddSnapshotFeature(DirectoryWithSnapshotFeature.DirectoryDiffList
diffs)
{
Preconditions.CheckState(!IsWithSnapshot(), "Directory is already with snapshot");
DirectoryWithSnapshotFeature sf = new DirectoryWithSnapshotFeature(diffs);
AddFeature(sf);
return(sf);
}
/// <summary>This method is usually called by the undo section of rename.</summary>
/// <remarks>
/// This method is usually called by the undo section of rename.
/// Before calling this function, in the rename operation, we replace the
/// original src node (of the rename operation) with a reference node (WithName
/// instance) in both the children list and a created list, delete the
/// reference node from the children list, and add it to the corresponding
/// deleted list.
/// To undo the above operations, we have the following steps in particular:
/// <pre>
/// 1) remove the WithName node from the deleted list (if it exists)
/// 2) replace the WithName node in the created list with srcChild
/// 3) add srcChild back as a child of srcParent. Note that we already add
/// the node into the created list of a snapshot diff in step 2, we do not need
/// to add srcChild to the created list of the latest snapshot.
/// </pre>
/// We do not need to update quota usage because the old child is in the
/// deleted list before.
/// </remarks>
/// <param name="oldChild">The reference node to be removed/replaced</param>
/// <param name="newChild">The node to be added back</param>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">should not throw this exception
/// </exception>
public virtual void UndoRename4ScrParent(INodeReference oldChild, INode newChild)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
Preconditions.CheckState(sf != null, "Directory does not have snapshot feature");
sf.GetDiffs().RemoveChild(Diff.ListType.Deleted, oldChild);
sf.GetDiffs().ReplaceChild(Diff.ListType.Created, oldChild, newChild);
AddChild(newChild, true, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
}
/// <summary>
/// <inheritDoc/>
/// <br/>
/// To destroy a DstReference node, we first remove its link with the
/// referred node. If the reference number of the referred node is <= 0, we
/// destroy the subtree of the referred node. Otherwise, we clean the
/// referred node's subtree and delete everything created after the last
/// rename operation, i.e., everything outside of the scope of the prior
/// WithName nodes.
/// </summary>
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot
int prior = GetPriorSnapshot(this);
// prior must be non-null, otherwise we do not have any previous
// WithName nodes, and the reference number will be 0.
Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId);
// identify the snapshot created after prior
int snapshot = GetSelfSnapshot(prior);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (referred.IsFile())
{
// if referred is a file, it must be a file with snapshot since we did
// recordModification before the rename
INodeFile file = referred.AsFile();
Preconditions.CheckState(file.IsWithSnapshot());
// make sure we mark the file as deleted
file.GetFileWithSnapshotFeature().DeleteCurrentFile();
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
}
else
{
if (referred.IsDirectory())
{
// similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot
INodeDirectory dir = referred.AsDirectory();
Preconditions.CheckState(dir.IsWithSnapshot());
try
{
DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks
, removedINodes);
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
}
/// <summary>Add quota usage for this inode excluding children.</summary>
public virtual QuotaCounts ComputeQuotaUsage4CurrentDirectory(BlockStoragePolicySuite
bsps, byte storagePolicyId, QuotaCounts counts)
{
counts.AddNameSpace(1);
// include the diff list
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf != null)
{
sf.ComputeQuotaUsage4CurrentDirectory(bsps, storagePolicyId, counts);
}
return(counts);
}
/// <summary>add DirectorySnapshottableFeature</summary>
public virtual void AddSnapshottableFeature()
{
Preconditions.CheckState(!IsSnapshottable(), "this is already snapshottable, this=%s"
, this);
DirectoryWithSnapshotFeature s = this.GetDirectoryWithSnapshotFeature();
DirectorySnapshottableFeature snapshottable = new DirectorySnapshottableFeature(s
);
if (s != null)
{
this.RemoveFeature(s);
}
this.AddFeature(snapshottable);
}
/// <summary>Remove the specified child from this directory.</summary>
public virtual bool RemoveChild(INode child, int latestSnapshotId)
{
if (IsInLatestSnapshot(latestSnapshotId))
{
// create snapshot feature if necessary
DirectoryWithSnapshotFeature sf = this.GetDirectoryWithSnapshotFeature();
if (sf == null)
{
sf = this.AddSnapshotFeature(null);
}
return(sf.RemoveChild(this, child, latestSnapshotId));
}
return(RemoveChild(child));
}
/// <summary>remove DirectorySnapshottableFeature</summary>
public virtual void RemoveSnapshottableFeature()
{
DirectorySnapshottableFeature s = GetDirectorySnapshottableFeature();
Preconditions.CheckState(s != null, "The dir does not have snapshottable feature: this=%s"
, this);
this.RemoveFeature(s);
if (s.GetDiffs().AsList().Count > 0)
{
// add a DirectoryWithSnapshotFeature back
DirectoryWithSnapshotFeature sf = new DirectoryWithSnapshotFeature(s.GetDiffs());
AddFeature(sf);
}
}
internal override void RecordModification(int latestSnapshotId)
{
if (IsInLatestSnapshot(latestSnapshotId) && !ShouldRecordInSrcSnapshot(latestSnapshotId
))
{
// add snapshot feature if necessary
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf == null)
{
sf = AddSnapshotFeature(null);
}
// record self in the diff list if necessary
sf.GetDiffs().SaveSelf2Snapshot(latestSnapshotId, this, null);
}
}
/// <summary>Save the child to the latest snapshot.</summary>
/// <returns>the child inode, which may be replaced.</returns>
public virtual INode SaveChild2Snapshot(INode child, int latestSnapshotId, INode
snapshotCopy)
{
if (latestSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.
CurrentStateId)
{
return(child);
}
// add snapshot feature if necessary
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf == null)
{
sf = this.AddSnapshotFeature(null);
}
return(sf.SaveChild2Snapshot(this, child, latestSnapshotId, snapshotCopy));
}
/// <summary>
/// Undo the rename operation for the dst tree, i.e., if the rename operation
/// (with OVERWRITE option) removes a file/dir from the dst tree, add it back
/// and delete possible record in the deleted list.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
public virtual void UndoRename4DstParent(BlockStoragePolicySuite bsps, INode deletedChild
, int latestSnapshotId)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
Preconditions.CheckState(sf != null, "Directory does not have snapshot feature");
bool removeDeletedChild = sf.GetDiffs().RemoveChild(Diff.ListType.Deleted, deletedChild
);
int sid = removeDeletedChild ? Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId : latestSnapshotId;
bool added = AddChild(deletedChild, true, sid);
// update quota usage if adding is successfully and the old child has not
// been stored in deleted list before
if (added && !removeDeletedChild)
{
QuotaCounts counts = deletedChild.ComputeQuotaUsage(bsps);
AddSpaceConsumed(counts, false);
}
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf != null)
{
sf.Clear(bsps, this, collectedBlocks, removedINodes);
}
foreach (INode child in GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId))
{
child.DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
if (GetAclFeature() != null)
{
AclStorage.RemoveAclFeature(GetAclFeature());
}
Clear();
removedINodes.AddItem(this);
}
public override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext
summary)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
if (sf != null)
{
sf.ComputeContentSummary4Snapshot(summary.GetBlockStoragePolicySuite(), summary.GetCounts
());
}
DirectoryWithQuotaFeature q = GetDirectoryWithQuotaFeature();
if (q != null)
{
return(q.ComputeContentSummary(this, summary));
}
else
{
return(ComputeDirectoryContentSummary(summary, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId));
}
}
/// <summary>
/// When destroying a reference node (WithName or DstReference), we call this
/// method to identify the snapshot which is the latest snapshot before the
/// reference node's creation.
/// </summary>
internal static int GetPriorSnapshot(Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeReference
@ref)
{
INodeReference.WithCount wc = (INodeReference.WithCount)@ref.GetReferredINode();
INodeReference.WithName wn = null;
if (@ref is INodeReference.DstReference)
{
wn = wc.GetLastWithName();
}
else
{
if (@ref is INodeReference.WithName)
{
wn = wc.GetPriorWithName((INodeReference.WithName)@ref);
}
}
if (wn != null)
{
INode referred = wc.GetReferredINode();
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
return(referred.AsFile().GetDiffs().GetPrior(wn.lastSnapshotId));
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
return(sf.GetDiffs().GetPrior(wn.lastSnapshotId));
}
}
}
}
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId);
}
private int GetSelfSnapshot()
{
INode referred = GetReferredINode().AsReference().GetReferredINode();
int snapshot = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId;
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
snapshot = referred.AsFile().GetDiffs().GetPrior(lastSnapshotId);
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
snapshot = sf.GetDiffs().GetPrior(lastSnapshotId);
}
}
}
return(snapshot);
}
public override INodeAttributes GetSnapshotINode(int snapshotId)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
return(sf == null ? this : sf.GetDiffs().GetSnapshotINode(snapshotId, this));
}
public override string ToDetailString()
{
DirectoryWithSnapshotFeature sf = this.GetDirectoryWithSnapshotFeature();
return(base.ToDetailString() + (sf == null ? string.Empty : ", " + sf.GetDiffs()));
}
public virtual DirectoryWithSnapshotFeature.DirectoryDiffList GetDiffs()
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
return(sf != null?sf.GetDiffs() : null);
}
