public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot , int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes ) { FileWithSnapshotFeature sf = GetFileWithSnapshotFeature(); if (sf != null) { return(sf.CleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, removedINodes )); } QuotaCounts counts = new QuotaCounts.Builder().Build(); if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId) { if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId) { // this only happens when deleting the current file and the file is not // in any snapshot ComputeQuotaUsage(bsps, counts, false); DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); } else { // when deleting the current file and the file is in snapshot, we should // clean the 0-sized block if the file is UC FileUnderConstructionFeature uc = GetFileUnderConstructionFeature(); if (uc != null) { uc.CleanZeroSizeBlock(this, collectedBlocks); } } } return(counts); }
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot , int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes ) { if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId && prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId) { QuotaCounts counts = new QuotaCounts.Builder().Build(); this.ComputeQuotaUsage(bsps, counts, true); DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); return(counts); } else { // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to // the previous WithName instance if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId) { prior = GetPriorSnapshot(this); } // if prior is not NO_SNAPSHOT_ID, and prior is not before the // to-be-deleted snapshot, we can quit here and leave the snapshot // deletion work to the src tree of rename if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId && prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId && Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator .Compare(snapshot, prior) <= 0) { return(new QuotaCounts.Builder().Build()); } return(GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes )); } }
/// <exception cref="System.Exception"/> public virtual void TestQuotaByStorageTypeWithSnapshot() { Path sub1 = new Path(dir, "Sub1"); dfs.Mkdirs(sub1); // Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1 dfs.SetStoragePolicy(sub1, HdfsConstants.OnessdStoragePolicyName); dfs.SetQuotaByStorageType(sub1, StorageType.Ssd, 4 * Blocksize); INode sub1Node = fsdir.GetINode4Write(sub1.ToString()); NUnit.Framework.Assert.IsTrue(sub1Node.IsDirectory()); NUnit.Framework.Assert.IsTrue(sub1Node.IsQuotaSet()); // Create file1 of size 2 * BLOCKSIZE under sub1 Path file1 = new Path(sub1, "file1"); long file1Len = 2 * Blocksize; DFSTestUtil.CreateFile(dfs, file1, file1Len, Replication, seed); // Create snapshot on sub1 named s1 SnapshotTestHelper.CreateSnapshot(dfs, sub1, "s1"); // Verify sub1 SSD usage is unchanged after creating snapshot s1 long ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed ().GetTypeSpaces().Get(StorageType.Ssd); NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed); // Delete file1 dfs.Delete(file1, false); // Verify sub1 SSD usage is unchanged due to the existence of snapshot s1 ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed ().GetTypeSpaces().Get(StorageType.Ssd); NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed); QuotaCounts counts1 = new QuotaCounts.Builder().Build(); sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts1 , true); NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), file1Len , counts1.GetTypeSpaces().Get(StorageType.Ssd)); ContentSummary cs1 = dfs.GetContentSummary(sub1); NUnit.Framework.Assert.AreEqual(cs1.GetSpaceConsumed(), file1Len * Replication); NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Ssd), file1Len); NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Disk), file1Len * 2); // Delete the snapshot s1 dfs.DeleteSnapshot(sub1, "s1"); // Verify sub1 SSD usage is fully reclaimed and changed to 0 ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed ().GetTypeSpaces().Get(StorageType.Ssd); NUnit.Framework.Assert.AreEqual(0, ssdConsumed); QuotaCounts counts2 = new QuotaCounts.Builder().Build(); sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts2 , true); NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), 0, counts2 .GetTypeSpaces().Get(StorageType.Ssd)); ContentSummary cs2 = dfs.GetContentSummary(sub1); NUnit.Framework.Assert.AreEqual(cs2.GetSpaceConsumed(), 0); NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Ssd), 0); NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Disk), 0); }
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId , int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes ) { DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature(); // there is snapshot data if (sf != null) { return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks , removedINodes)); } // there is no snapshot data if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId && snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId) { // destroy the whole subtree and collect blocks that should be deleted QuotaCounts counts = new QuotaCounts.Builder().Build(); this.ComputeQuotaUsage(bsps, counts, true); DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes); return(counts); } else { // process recursively down the subtree QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks , removedINodes, null); if (IsQuotaSet()) { GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation()); } return(counts); } }
/// <summary>Call cleanSubtree(..) recursively down the subtree.</summary> public virtual QuotaCounts CleanSubtreeRecursively(BlockStoragePolicySuite bsps, int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes, IDictionary <INode, INode> excludedNodes) { QuotaCounts counts = new QuotaCounts.Builder().Build(); // in case of deletion snapshot, since this call happens after we modify // the diff list, the snapshot to be deleted has been combined or renamed // to its latest previous snapshot. (besides, we also need to consider nodes // created after prior but before snapshot. this will be done in // DirectoryWithSnapshotFeature) int s = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId && prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId ? prior : snapshot; foreach (INode child in GetChildrenList(s)) { if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId && excludedNodes != null && excludedNodes.Contains(child)) { continue; } else { QuotaCounts childCounts = child.CleanSubtree(bsps, snapshot, prior, collectedBlocks , removedINodes); counts.Add(childCounts); } } return(counts); }
/// <summary> /// Returns a QuotaCounts whose value is /// <c>(-this)</c> /// . /// </summary> /// <returns> /// /// <c>-this</c> /// </returns> public virtual QuotaCounts Negation() { QuotaCounts ret = new QuotaCounts.Builder().QuotaCount(this).Build(); ret.nsSsCounts.Negation(); ret.tsCounts.Negation(); return(ret); }
public sealed override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext summary) { //only count storagespace for WithName QuotaCounts q = new QuotaCounts.Builder().Build(); ComputeQuotaUsage(summary.GetBlockStoragePolicySuite(), GetStoragePolicyID(), q, false, lastSnapshotId); summary.GetCounts().AddContent(Content.Diskspace, q.GetStorageSpace()); summary.GetCounts().AddTypeSpaces(q.GetTypeSpaces()); return(summary); }
public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot , int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes ) { // since WithName node resides in deleted list acting as a snapshot copy, // the parameter snapshot must be non-null Preconditions.CheckArgument(snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId); // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the // previous WithName instance if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId) { prior = GetPriorSnapshot(this); } if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId && Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator .Compare(snapshot, prior) <= 0) { return(new QuotaCounts.Builder().Build()); } QuotaCounts counts = GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks , removedINodes); INodeReference @ref = GetReferredINode().GetParentReference(); if (@ref != null) { try { @ref.AddSpaceConsumed(counts.Negation(), true); } catch (QuotaExceededException) { Org.Mortbay.Log.Log.Warn("Should not have QuotaExceededException"); } } if (snapshot < lastSnapshotId) { // for a WithName node, when we compute its quota usage, we only count // in all the nodes existing at the time of the corresponding rename op. // Thus if we are deleting a snapshot before/at the snapshot associated // with lastSnapshotId, we do not need to update the quota upwards. counts = new QuotaCounts.Builder().Build(); } return(counts); }
/// <exception cref="System.Exception"/> public virtual void TestQuotaByStorageTypeWithFileCreateDelete() { Path foo = new Path(dir, "foo"); Path createdFile1 = new Path(foo, "created_file1.data"); dfs.Mkdirs(foo); dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName); // set quota by storage type on directory "foo" dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10); INode fnode = fsdir.GetINode4Write(foo.ToString()); NUnit.Framework.Assert.IsTrue(fnode.IsDirectory()); NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet()); // Create file of size 2.5 * BLOCKSIZE under directory "foo" long file1Len = Blocksize * 2 + Blocksize / 2; int bufLen = Blocksize / 16; DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication , seed); // Verify space consumed and remaining quota long storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed ().GetTypeSpaces().Get(StorageType.Ssd); NUnit.Framework.Assert.AreEqual(file1Len, storageTypeConsumed); // Delete file and verify the consumed space of the storage type is updated dfs.Delete(createdFile1, false); storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed ().GetTypeSpaces().Get(StorageType.Ssd); NUnit.Framework.Assert.AreEqual(0, storageTypeConsumed); QuotaCounts counts = new QuotaCounts.Builder().Build(); fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true ); NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts .GetTypeSpaces().Get(StorageType.Ssd)); ContentSummary cs = dfs.GetContentSummary(foo); NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), 0); NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), 0); NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), 0); }
/// <summary> /// Both traditional space quota and the storage type quota for SSD are set and /// not exceeded. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestQuotaByStorageTypeWithTraditionalQuota() { Path foo = new Path(dir, "foo"); dfs.Mkdirs(foo); dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName); dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10); dfs.SetQuota(foo, long.MaxValue - 1, Replication * Blocksize * 10); INode fnode = fsdir.GetINode4Write(foo.ToString()); NUnit.Framework.Assert.IsTrue(fnode.IsDirectory()); NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet()); Path createdFile = new Path(foo, "created_file.data"); long fileLen = Blocksize * 2 + Blocksize / 2; DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication , seed); QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed (); NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace()); NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace()); dfs.Delete(createdFile, true); QuotaCounts cntAfterDelete = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed (); NUnit.Framework.Assert.AreEqual(1, cntAfterDelete.GetNameSpace()); NUnit.Framework.Assert.AreEqual(0, cntAfterDelete.GetStorageSpace()); // Validate the computeQuotaUsage() QuotaCounts counts = new QuotaCounts.Builder().Build(); fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true ); NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 1, counts .GetNameSpace()); NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts .GetStorageSpace()); }
private static QuotaCounts ComputeQuotaDeltas(FSDirectory fsd, INodeFile target, INodeFile[] srcList) { QuotaCounts deltas = new QuotaCounts.Builder().Build(); short targetRepl = target.GetBlockReplication(); foreach (INodeFile src in srcList) { short srcRepl = src.GetBlockReplication(); long fileSize = src.ComputeFileSize(); if (targetRepl != srcRepl) { deltas.AddStorageSpace(fileSize * (targetRepl - srcRepl)); BlockStoragePolicy bsp = fsd.GetBlockStoragePolicySuite().GetPolicy(src.GetStoragePolicyID ()); if (bsp != null) { IList <StorageType> srcTypeChosen = bsp.ChooseStorageTypes(srcRepl); foreach (StorageType t in srcTypeChosen) { if (t.SupportTypeQuota()) { deltas.AddTypeSpace(t, -fileSize); } } IList <StorageType> targetTypeChosen = bsp.ChooseStorageTypes(targetRepl); foreach (StorageType t_1 in targetTypeChosen) { if (t_1.SupportTypeQuota()) { deltas.AddTypeSpace(t_1, fileSize); } } } } } return(deltas); }
private QuotaCounts(QuotaCounts.Builder builder) { this.nsSsCounts = builder.nsSsCounts; this.tsCounts = builder.tsCounts; }