public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// there is snapshot data
if (sf != null)
{
return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes));
}
// there is no snapshot data
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
// destroy the whole subtree and collect blocks that should be deleted
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// process recursively down the subtree
QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes, null);
if (IsQuotaSet())
{
GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation());
}
return(counts);
}
}
internal override QuotaCounts DestroyDiffAndCollectBlocks(BlockStoragePolicySuite
bsps, INodeFile currentINode, INode.BlocksMapUpdateInfo collectedBlocks, IList <
INode> removedINodes)
{
return(currentINode.GetFileWithSnapshotFeature().UpdateQuotaAndCollectBlocks(bsps
, currentINode, this, collectedBlocks, removedINodes));
}
/// <summary>Call cleanSubtree(..) recursively down the subtree.</summary>
public virtual QuotaCounts CleanSubtreeRecursively(BlockStoragePolicySuite bsps,
int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode>
removedINodes, IDictionary <INode, INode> excludedNodes)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
// in case of deletion snapshot, since this call happens after we modify
// the diff list, the snapshot to be deleted has been combined or renamed
// to its latest previous snapshot. (besides, we also need to consider nodes
// created after prior but before snapshot. this will be done in
// DirectoryWithSnapshotFeature)
int s = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId
? prior : snapshot;
foreach (INode child in GetChildrenList(s))
{
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
excludedNodes != null && excludedNodes.Contains(child))
{
continue;
}
else
{
QuotaCounts childCounts = child.CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
counts.Add(childCounts);
}
}
return(counts);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to
// the previous WithName instance
if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
prior = GetPriorSnapshot(this);
}
// if prior is not NO_SNAPSHOT_ID, and prior is not before the
// to-be-deleted snapshot, we can quit here and leave the snapshot
// deletion work to the src tree of rename
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
.Compare(snapshot, prior) <= 0)
{
return(new QuotaCounts.Builder().Build());
}
return(GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes
));
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/>
public virtual Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot RemoveSnapshot
(BlockStoragePolicySuite bsps, string snapshotName, INode.BlocksMapUpdateInfo collectedBlocks
, IList <INode> removedINodes)
{
return(GetDirectorySnapshottableFeature().RemoveSnapshot(bsps, this, snapshotName
, collectedBlocks, removedINodes));
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (blocks != null && collectedBlocks != null)
{
foreach (BlockInfoContiguous blk in blocks)
{
collectedBlocks.AddDeleteBlock(blk);
blk.SetBlockCollection(null);
}
}
SetBlocks(null);
if (GetAclFeature() != null)
{
AclStorage.RemoveAclFeature(GetAclFeature());
}
Clear();
removedINodes.AddItem(this);
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
sf.GetDiffs().DestroyAndCollectSnapshotBlocks(collectedBlocks);
sf.ClearDiffs();
}
}
/// <summary>
/// Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
/// dstInodes[dstInodes.length-1]
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
private static void VerifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath
dst)
{
if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
{
// Do not check quota if edits log is still being processed
return;
}
int i = 0;
while (src.GetINode(i) == dst.GetINode(i))
{
i++;
}
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
QuotaCounts delta = src.GetLastINode().ComputeQuotaUsage(bsps);
// Reduce the required quota by dst that is being removed
INode dstINode = dst.GetLastINode();
if (dstINode != null)
{
delta.Subtract(dstINode.ComputeQuotaUsage(bsps));
}
FSDirectory.VerifyQuota(dst, dst.Length() - 1, delta, src.GetINode(i - 1));
}
public virtual void TestMultipleWarmsInDifferentOrder()
{
BlockStoragePolicySuite bsps = BlockStoragePolicySuite.CreateDefaultSuite();
StoragePolicySummary sts = new StoragePolicySummary(bsps.GetAllPolicies());
BlockStoragePolicy warm = bsps.GetPolicy("WARM");
//DISK:1,ARCHIVE:1
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk }, warm);
//DISK:2,ARCHIVE:1
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk, StorageType.Disk }, warm);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive, StorageType.Disk }, warm);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Archive }, warm);
//DISK:1,ARCHIVE:2
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk }, warm);
//DISK:2,ARCHIVE:2
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk, StorageType.Disk }, warm);
IDictionary <string, long> actualOutput = ConvertToStringMap(sts);
NUnit.Framework.Assert.AreEqual(4, actualOutput.Count);
IDictionary <string, long> expectedOutput = new Dictionary <string, long>();
expectedOutput["WARM|DISK:1,ARCHIVE:1(WARM)"] = 2l;
expectedOutput["WARM|DISK:2,ARCHIVE:1"] = 3l;
expectedOutput["WARM|DISK:1,ARCHIVE:2(WARM)"] = 3l;
expectedOutput["WARM|DISK:2,ARCHIVE:2"] = 1l;
NUnit.Framework.Assert.AreEqual(expectedOutput, actualOutput);
}
public override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps, byte
blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// we are computing the quota usage for a specific snapshot here, i.e., the
// computation only includes files/directories that exist at the time of the
// given snapshot
if (sf != null && lastSnapshotId != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId && !(useCache && IsQuotaSet()))
{
ReadOnlyList <INode> childrenList = GetChildrenList(lastSnapshotId);
foreach (INode child in childrenList)
{
byte childPolicyId = child.GetStoragePolicyIDForQuota(blockStoragePolicyId);
child.ComputeQuotaUsage(bsps, childPolicyId, counts, useCache, lastSnapshotId);
}
counts.AddNameSpace(1);
return(counts);
}
// compute the quota usage in the scope of the current directory tree
DirectoryWithQuotaFeature q = GetDirectoryWithQuotaFeature();
if (useCache && q != null && q.IsQuotaSet())
{
// use the cached quota
return(q.AddCurrentSpaceUsage(counts));
}
else
{
useCache = q != null && !q.IsQuotaSet() ? false : useCache;
return(ComputeDirectoryQuotaUsage(bsps, blockStoragePolicyId, counts, useCache, lastSnapshotId
));
}
}
public virtual void TestSetAndGetStoragePolicy()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(fs, bar, Size, Repl, 0);
StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0, "Set storage policy WARM on "
+ foo.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD", 0, "Set storage policy COLD on "
+ bar.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /fooz -policy WARM", 2, "File/Directory does not exist: /fooz"
);
BlockStoragePolicySuite suite = BlockStoragePolicySuite.CreateDefaultSuite();
BlockStoragePolicy warm = suite.GetPolicy("WARM");
BlockStoragePolicy cold = suite.GetPolicy("COLD");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + ":\n" + warm);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + ":\n" + cold);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /fooz", 2, "File/Directory does not exist: /fooz"
);
}
public virtual void TestMultipleHotsWithDifferentCounts()
{
BlockStoragePolicySuite bsps = BlockStoragePolicySuite.CreateDefaultSuite();
StoragePolicySummary sts = new StoragePolicySummary(bsps.GetAllPolicies());
BlockStoragePolicy hot = bsps.GetPolicy("HOT");
sts.Add(new StorageType[] { StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk,
StorageType.Disk }, hot);
IDictionary <string, long> actualOutput = ConvertToStringMap(sts);
NUnit.Framework.Assert.AreEqual(4, actualOutput.Count);
IDictionary <string, long> expectedOutput = new Dictionary <string, long>();
expectedOutput["HOT|DISK:1(HOT)"] = 1l;
expectedOutput["HOT|DISK:2(HOT)"] = 2l;
expectedOutput["HOT|DISK:3(HOT)"] = 2l;
expectedOutput["HOT|DISK:4(HOT)"] = 1l;
NUnit.Framework.Assert.AreEqual(expectedOutput, actualOutput);
}
/// <summary>
/// If some blocks at the end of the block list no longer belongs to
/// any inode, collect them and update the block list.
/// </summary>
public virtual void CollectBlocksAndClear(BlockStoragePolicySuite bsps, INodeFile
file, INode.BlocksMapUpdateInfo info, IList <INode> removedINodes)
{
// check if everything is deleted.
if (IsCurrentFileDeleted() && GetDiffs().AsList().IsEmpty())
{
file.DestroyAndCollectBlocks(bsps, info, removedINodes);
return;
}
// find max file size.
long max;
FileDiff diff = GetDiffs().GetLast();
if (IsCurrentFileDeleted())
{
max = diff == null ? 0 : diff.GetFileSize();
}
else
{
max = file.ComputeFileSize();
}
// Collect blocks that should be deleted
FileDiff last = diffs.GetLast();
BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.GetBlocks();
if (snapshotBlocks == null)
{
file.CollectBlocksBeyondMax(max, info);
}
else
{
file.CollectBlocksBeyondSnapshot(snapshotBlocks, info);
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
return(sf.CleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, removedINodes
));
}
QuotaCounts counts = new QuotaCounts.Builder().Build();
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
// this only happens when deleting the current file and the file is not
// in any snapshot
ComputeQuotaUsage(bsps, counts, false);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// when deleting the current file and the file is in snapshot, we should
// clean the 0-sized block if the file is UC
FileUnderConstructionFeature uc = GetFileUnderConstructionFeature();
if (uc != null)
{
uc.CleanZeroSizeBlock(this, collectedBlocks);
}
}
}
return(counts);
}
public _Processor_218(BlockStoragePolicySuite bsps, QuotaCounts counts, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
this.bsps = bsps;
this.counts = counts;
this.collectedBlocks = collectedBlocks;
this.removedINodes = removedINodes;
}
public virtual ContentSummaryComputationContext ComputeContentSummary(BlockStoragePolicySuite
bsps, INodeDirectory snapshotRoot, ContentSummaryComputationContext summary)
{
snapshotRoot.ComputeContentSummary(summary);
summary.GetCounts().AddContent(Content.Snapshot, snapshotsByNames.Count);
summary.GetCounts().AddContent(Content.SnapshottableDirectory, 1);
return(summary);
}
/// <summary>
/// Count subtree
/// <see cref="Quota.Namespace"/>
/// and
/// <see cref="Quota.Storagespace"/>
/// usages.
/// Entry point for FSDirectory where blockStoragePolicyId is given its initial
/// value.
/// </summary>
public QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps)
{
byte storagePolicyId = IsSymlink() ? BlockStoragePolicySuite.IdUnspecified : GetStoragePolicyID
();
return(ComputeQuotaUsage(bsps, storagePolicyId, new QuotaCounts.Builder().Build()
, true, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId));
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
// used by WithCount
return(referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes
));
}
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
// used by WithCount
if (RemoveReference(this) <= 0)
{
referred.DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
}
internal MigrationTest(TestStorageMover _enclosing, TestStorageMover.ClusterScheme
cScheme, TestStorageMover.NamespaceScheme nsScheme)
{
this._enclosing = _enclosing;
this.clusterScheme = cScheme;
this.nsScheme = nsScheme;
this.conf = this.clusterScheme.conf;
this.policies = TestStorageMover.DefaultPolicies;
}
public QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps, QuotaCounts counts
, bool useCache)
{
byte storagePolicyId = IsSymlink() ? BlockStoragePolicySuite.IdUnspecified : GetStoragePolicyID
();
return(ComputeQuotaUsage(bsps, storagePolicyId, counts, useCache, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId));
}
// This is the only place that needs to use the BlockStoragePolicySuite to
// derive the intended storage type usage for quota by storage type
public sealed override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps
, byte blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId
)
{
long nsDelta = 1;
long ssDeltaNoReplication;
short replication;
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
FileDiffList fileDiffList = sf.GetDiffs();
int last = fileDiffList.GetLastSnapshotId();
if (lastSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ||
last == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
else
{
if (last < lastSnapshotId)
{
ssDeltaNoReplication = ComputeFileSize(true, false);
replication = GetFileReplication();
}
else
{
int sid = fileDiffList.GetSnapshotById(lastSnapshotId);
ssDeltaNoReplication = StoragespaceConsumedNoReplication(sid);
replication = GetReplication(sid);
}
}
}
else
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
counts.AddNameSpace(nsDelta);
counts.AddStorageSpace(ssDeltaNoReplication * replication);
if (blockStoragePolicyId != BlockStoragePolicySuite.IdUnspecified)
{
BlockStoragePolicy bsp = bsps.GetPolicy(blockStoragePolicyId);
IList <StorageType> storageTypes = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in storageTypes)
{
if (!t.SupportTypeQuota())
{
continue;
}
counts.AddTypeSpace(t, ssDeltaNoReplication);
}
}
return(counts);
}
/// <exception cref="System.IO.IOException"/>
private static void SetDirStoragePolicy(FSDirectory fsd, INodeDirectory inode, byte
policyId, int latestSnapshotId)
{
IList <XAttr> existingXAttrs = XAttrStorage.ReadINodeXAttrs(inode);
XAttr xAttr = BlockStoragePolicySuite.BuildXAttr(policyId);
IList <XAttr> newXAttrs = FSDirXAttrOp.SetINodeXAttrs(fsd, existingXAttrs, Arrays.
AsList(xAttr), EnumSet.Of(XAttrSetFlag.Create, XAttrSetFlag.Replace));
XAttrStorage.UpdateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
internal override QuotaCounts CombinePosteriorAndCollectBlocks(BlockStoragePolicySuite
bsps, INodeDirectory currentDir, DirectoryWithSnapshotFeature.DirectoryDiff posterior
, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
diff.CombinePosterior(posterior.diff, new _Processor_218(bsps, counts, collectedBlocks
, removedINodes));
return(counts);
}
public virtual void Clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode
, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes)
{
// destroy its diff list
foreach (DirectoryWithSnapshotFeature.DirectoryDiff diff in diffs)
{
diff.DestroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks, removedINodes
);
}
diffs.Clear();
}
internal override QuotaCounts CombinePosteriorAndCollectBlocks(BlockStoragePolicySuite
bsps, INodeFile currentINode, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.FileDiff
posterior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
FileWithSnapshotFeature sf = currentINode.GetFileWithSnapshotFeature();
System.Diagnostics.Debug.Assert(sf != null, "FileWithSnapshotFeature is null");
return(sf.UpdateQuotaAndCollectBlocks(bsps, currentINode, posterior, collectedBlocks
, removedINodes));
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
internal RenameOperation(FSDirectory fsd, string src, string dst, INodesInPath srcIIP
, INodesInPath dstIIP)
{
this.fsd = fsd;
this.src = src;
this.dst = dst;
this.srcIIP = srcIIP;
this.dstIIP = dstIIP;
this.srcParentIIP = srcIIP.GetParentINodesInPath();
this.dstParentIIP = dstIIP.GetParentINodesInPath();
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
srcChild = this.srcIIP.GetLastINode();
srcChildName = srcChild.GetLocalNameBytes();
int srcLatestSnapshotId = srcIIP.GetLatestSnapshotId();
isSrcInSnapshot = srcChild.IsInLatestSnapshot(srcLatestSnapshotId);
srcChildIsReference = srcChild.IsReference();
srcParent = this.srcIIP.GetINode(-2).AsDirectory();
// Record the snapshot on srcChild. After the rename, before any new
// snapshot is taken on the dst tree, changes will be recorded in the
// latest snapshot of the src tree.
if (isSrcInSnapshot)
{
srcChild.RecordModification(srcLatestSnapshotId);
}
// check srcChild for reference
srcRefDstSnapshot = srcChildIsReference ? srcChild.AsReference().GetDstSnapshotId
() : Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId;
oldSrcCounts = new QuotaCounts.Builder().Build();
if (isSrcInSnapshot)
{
INodeReference.WithName withName = srcParent.ReplaceChild4ReferenceWithName(srcChild
, srcLatestSnapshotId);
withCount = (INodeReference.WithCount)withName.GetReferredINode();
srcChild = withName;
this.srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, srcChild);
// get the counts before rename
withCount.GetReferredINode().ComputeQuotaUsage(bsps, oldSrcCounts, true);
}
else
{
if (srcChildIsReference)
{
// srcChild is reference but srcChild is not in latest snapshot
withCount = (INodeReference.WithCount)srcChild.AsReference().GetReferredINode();
}
else
{
withCount = null;
}
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
internal virtual void UpdateQuotasInSourceTree(BlockStoragePolicySuite bsps)
{
// update the quota usage in src tree
if (isSrcInSnapshot)
{
// get the counts after rename
QuotaCounts newSrcCounts = srcChild.ComputeQuotaUsage(bsps, new QuotaCounts.Builder
().Build(), false);
newSrcCounts.Subtract(oldSrcCounts);
srcParent.AddSpaceConsumed(newSrcCounts, false);
}
}
/// <summary>
/// <inheritDoc/>
/// <br/>
/// To destroy a DstReference node, we first remove its link with the
/// referred node. If the reference number of the referred node is <= 0, we
/// destroy the subtree of the referred node. Otherwise, we clean the
/// referred node's subtree and delete everything created after the last
/// rename operation, i.e., everything outside of the scope of the prior
/// WithName nodes.
/// </summary>
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot
int prior = GetPriorSnapshot(this);
// prior must be non-null, otherwise we do not have any previous
// WithName nodes, and the reference number will be 0.
Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId);
// identify the snapshot created after prior
int snapshot = GetSelfSnapshot(prior);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (referred.IsFile())
{
// if referred is a file, it must be a file with snapshot since we did
// recordModification before the rename
INodeFile file = referred.AsFile();
Preconditions.CheckState(file.IsWithSnapshot());
// make sure we mark the file as deleted
file.GetFileWithSnapshotFeature().DeleteCurrentFile();
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
}
else
{
if (referred.IsDirectory())
{
// similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot
INodeDirectory dir = referred.AsDirectory();
Preconditions.CheckState(dir.IsWithSnapshot());
try
{
DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks
, removedINodes);
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
if (snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.
NoSnapshotId)
{
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
return(new QuotaCounts.Builder().NameSpace(1).Build());
}
static TestStorageMover()
{
DefaultConf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
DefaultConf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
DefaultConf.SetLong(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 2L);
DefaultConf.SetLong(DFSConfigKeys.DfsMoverMovedwinwidthKey, 2000L);
DefaultPolicies = BlockStoragePolicySuite.CreateDefaultSuite();
Hot = DefaultPolicies.GetPolicy(HdfsConstants.HotStoragePolicyName);
Warm = DefaultPolicies.GetPolicy(HdfsConstants.WarmStoragePolicyName);
Cold = DefaultPolicies.GetPolicy(HdfsConstants.ColdStoragePolicyName);
TestBalancer.InitTestSetup();
Dispatcher.SetDelayAfterErrors(1000L);
}
