public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
return(sf.CleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, removedINodes
));
}
QuotaCounts counts = new QuotaCounts.Builder().Build();
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
// this only happens when deleting the current file and the file is not
// in any snapshot
ComputeQuotaUsage(bsps, counts, false);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// when deleting the current file and the file is in snapshot, we should
// clean the 0-sized block if the file is UC
FileUnderConstructionFeature uc = GetFileUnderConstructionFeature();
if (uc != null)
{
uc.CleanZeroSizeBlock(this, collectedBlocks);
}
}
}
return(counts);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to
// the previous WithName instance
if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
prior = GetPriorSnapshot(this);
}
// if prior is not NO_SNAPSHOT_ID, and prior is not before the
// to-be-deleted snapshot, we can quit here and leave the snapshot
// deletion work to the src tree of rename
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
.Compare(snapshot, prior) <= 0)
{
return(new QuotaCounts.Builder().Build());
}
return(GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes
));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithSnapshot()
{
Path sub1 = new Path(dir, "Sub1");
dfs.Mkdirs(sub1);
// Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1
dfs.SetStoragePolicy(sub1, HdfsConstants.OnessdStoragePolicyName);
dfs.SetQuotaByStorageType(sub1, StorageType.Ssd, 4 * Blocksize);
INode sub1Node = fsdir.GetINode4Write(sub1.ToString());
NUnit.Framework.Assert.IsTrue(sub1Node.IsDirectory());
NUnit.Framework.Assert.IsTrue(sub1Node.IsQuotaSet());
// Create file1 of size 2 * BLOCKSIZE under sub1
Path file1 = new Path(sub1, "file1");
long file1Len = 2 * Blocksize;
DFSTestUtil.CreateFile(dfs, file1, file1Len, Replication, seed);
// Create snapshot on sub1 named s1
SnapshotTestHelper.CreateSnapshot(dfs, sub1, "s1");
// Verify sub1 SSD usage is unchanged after creating snapshot s1
long ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// Delete file1
dfs.Delete(file1, false);
// Verify sub1 SSD usage is unchanged due to the existence of snapshot s1
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
QuotaCounts counts1 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts1
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), file1Len
, counts1.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs1 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs1.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Disk), file1Len *
2);
// Delete the snapshot s1
dfs.DeleteSnapshot(sub1, "s1");
// Verify sub1 SSD usage is fully reclaimed and changed to 0
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, ssdConsumed);
QuotaCounts counts2 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts2
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), 0, counts2
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs2 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs2.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Disk), 0);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
, int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();
// there is snapshot data
if (sf != null)
{
return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes));
}
// there is no snapshot data
if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
// destroy the whole subtree and collect blocks that should be deleted
QuotaCounts counts = new QuotaCounts.Builder().Build();
this.ComputeQuotaUsage(bsps, counts, true);
DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return(counts);
}
else
{
// process recursively down the subtree
QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks
, removedINodes, null);
if (IsQuotaSet())
{
GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation());
}
return(counts);
}
}
/// <summary>Call cleanSubtree(..) recursively down the subtree.</summary>
public virtual QuotaCounts CleanSubtreeRecursively(BlockStoragePolicySuite bsps,
int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode>
removedINodes, IDictionary <INode, INode> excludedNodes)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
// in case of deletion snapshot, since this call happens after we modify
// the diff list, the snapshot to be deleted has been combined or renamed
// to its latest previous snapshot. (besides, we also need to consider nodes
// created after prior but before snapshot. this will be done in
// DirectoryWithSnapshotFeature)
int s = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId
? prior : snapshot;
foreach (INode child in GetChildrenList(s))
{
if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
excludedNodes != null && excludedNodes.Contains(child))
{
continue;
}
else
{
QuotaCounts childCounts = child.CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
counts.Add(childCounts);
}
}
return(counts);
}
/// <summary>
/// Returns a QuotaCounts whose value is
/// <c>(-this)</c>
/// .
/// </summary>
/// <returns>
///
/// <c>-this</c>
/// </returns>
public virtual QuotaCounts Negation()
{
QuotaCounts ret = new QuotaCounts.Builder().QuotaCount(this).Build();
ret.nsSsCounts.Negation();
ret.tsCounts.Negation();
return(ret);
}
public sealed override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext
summary)
{
//only count storagespace for WithName
QuotaCounts q = new QuotaCounts.Builder().Build();
ComputeQuotaUsage(summary.GetBlockStoragePolicySuite(), GetStoragePolicyID(), q,
false, lastSnapshotId);
summary.GetCounts().AddContent(Content.Diskspace, q.GetStorageSpace());
summary.GetCounts().AddTypeSpaces(q.GetTypeSpaces());
return(summary);
}
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
)
{
// since WithName node resides in deleted list acting as a snapshot copy,
// the parameter snapshot must be non-null
Preconditions.CheckArgument(snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
// previous WithName instance
if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
{
prior = GetPriorSnapshot(this);
}
if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
.Compare(snapshot, prior) <= 0)
{
return(new QuotaCounts.Builder().Build());
}
QuotaCounts counts = GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks
, removedINodes);
INodeReference @ref = GetReferredINode().GetParentReference();
if (@ref != null)
{
try
{
@ref.AddSpaceConsumed(counts.Negation(), true);
}
catch (QuotaExceededException)
{
Org.Mortbay.Log.Log.Warn("Should not have QuotaExceededException");
}
}
if (snapshot < lastSnapshotId)
{
// for a WithName node, when we compute its quota usage, we only count
// in all the nodes existing at the time of the corresponding rename op.
// Thus if we are deleting a snapshot before/at the snapshot associated
// with lastSnapshotId, we do not need to update the quota upwards.
counts = new QuotaCounts.Builder().Build();
}
return(counts);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateDelete()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2.5 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2 + Blocksize / 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, storageTypeConsumed);
// Delete file and verify the consumed space of the storage type is updated
dfs.Delete(createdFile1, false);
storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, storageTypeConsumed);
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), 0);
}
/// <summary>
/// Both traditional space quota and the storage type quota for SSD are set and
/// not exceeded.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithTraditionalQuota()
{
Path foo = new Path(dir, "foo");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
dfs.SetQuota(foo, long.MaxValue - 1, Replication * Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
Path createdFile = new Path(foo, "created_file.data");
long fileLen = Blocksize * 2 + Blocksize / 2;
DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication
, seed);
QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
();
NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace());
NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace());
dfs.Delete(createdFile, true);
QuotaCounts cntAfterDelete = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
();
NUnit.Framework.Assert.AreEqual(1, cntAfterDelete.GetNameSpace());
NUnit.Framework.Assert.AreEqual(0, cntAfterDelete.GetStorageSpace());
// Validate the computeQuotaUsage()
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 1, counts
.GetNameSpace());
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetStorageSpace());
}
private static QuotaCounts ComputeQuotaDeltas(FSDirectory fsd, INodeFile target,
INodeFile[] srcList)
{
QuotaCounts deltas = new QuotaCounts.Builder().Build();
short targetRepl = target.GetBlockReplication();
foreach (INodeFile src in srcList)
{
short srcRepl = src.GetBlockReplication();
long fileSize = src.ComputeFileSize();
if (targetRepl != srcRepl)
{
deltas.AddStorageSpace(fileSize * (targetRepl - srcRepl));
BlockStoragePolicy bsp = fsd.GetBlockStoragePolicySuite().GetPolicy(src.GetStoragePolicyID
());
if (bsp != null)
{
IList <StorageType> srcTypeChosen = bsp.ChooseStorageTypes(srcRepl);
foreach (StorageType t in srcTypeChosen)
{
if (t.SupportTypeQuota())
{
deltas.AddTypeSpace(t, -fileSize);
}
}
IList <StorageType> targetTypeChosen = bsp.ChooseStorageTypes(targetRepl);
foreach (StorageType t_1 in targetTypeChosen)
{
if (t_1.SupportTypeQuota())
{
deltas.AddTypeSpace(t_1, fileSize);
}
}
}
}
}
return(deltas);
}
private QuotaCounts(QuotaCounts.Builder builder)
{
this.nsSsCounts = builder.nsSsCounts;
this.tsCounts = builder.tsCounts;
}
