Example #1
0
        /// <summary>
        /// Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
        /// dstInodes[dstInodes.length-1]
        /// </summary>
        /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
        private static void VerifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath
                                                 dst)
        {
            if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
            {
                // Do not check quota if edits log is still being processed
                return;
            }
            int i = 0;

            while (src.GetINode(i) == dst.GetINode(i))
            {
                i++;
            }
            // src[i - 1] is the last common ancestor.
            BlockStoragePolicySuite bsps  = fsd.GetBlockStoragePolicySuite();
            QuotaCounts             delta = src.GetLastINode().ComputeQuotaUsage(bsps);
            // Reduce the required quota by dst that is being removed
            INode dstINode = dst.GetLastINode();

            if (dstINode != null)
            {
                delta.Subtract(dstINode.ComputeQuotaUsage(bsps));
            }
            FSDirectory.VerifyQuota(dst, dst.Length() - 1, delta, src.GetINode(i - 1));
        }
Example #2
0
        /// <summary>Call cleanSubtree(..) recursively down the subtree.</summary>
        public virtual QuotaCounts CleanSubtreeRecursively(BlockStoragePolicySuite bsps,
                                                           int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode>
                                                           removedINodes, IDictionary <INode, INode> excludedNodes)
        {
            QuotaCounts counts = new QuotaCounts.Builder().Build();
            // in case of deletion snapshot, since this call happens after we modify
            // the diff list, the snapshot to be deleted has been combined or renamed
            // to its latest previous snapshot. (besides, we also need to consider nodes
            // created after prior but before snapshot. this will be done in
            // DirectoryWithSnapshotFeature)
            int s = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
                    prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId
                                 ? prior : snapshot;

            foreach (INode child in GetChildrenList(s))
            {
                if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
                    excludedNodes != null && excludedNodes.Contains(child))
                {
                    continue;
                }
                else
                {
                    QuotaCounts childCounts = child.CleanSubtree(bsps, snapshot, prior, collectedBlocks
                                                                 , removedINodes);
                    counts.Add(childCounts);
                }
            }
            return(counts);
        }
Example #3
0
        public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshotId
                                                 , int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
                                                 )
        {
            DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();

            // there is snapshot data
            if (sf != null)
            {
                return(sf.CleanDirectory(bsps, this, snapshotId, priorSnapshotId, collectedBlocks
                                         , removedINodes));
            }
            // there is no snapshot data
            if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
                snapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
            {
                // destroy the whole subtree and collect blocks that should be deleted
                QuotaCounts counts = new QuotaCounts.Builder().Build();
                this.ComputeQuotaUsage(bsps, counts, true);
                DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
                return(counts);
            }
            else
            {
                // process recursively down the subtree
                QuotaCounts counts = CleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, collectedBlocks
                                                             , removedINodes, null);
                if (IsQuotaSet())
                {
                    GetDirectoryWithQuotaFeature().AddSpaceConsumed2Cache(counts.Negation());
                }
                return(counts);
            }
        }
Example #4
0
            private void LoadRootINode(FsImageProto.INodeSection.INode p)
            {
                INodeDirectory root    = LoadINodeDirectory(p, parent.GetLoaderContext());
                QuotaCounts    q       = root.GetQuotaCounts();
                long           nsQuota = q.GetNameSpace();
                long           dsQuota = q.GetStorageSpace();

                if (nsQuota != -1 || dsQuota != -1)
                {
                    dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(nsQuota, dsQuota);
                }
                EnumCounters <StorageType> typeQuotas = q.GetTypeSpaces();

                if (typeQuotas.AnyGreaterOrEqual(0))
                {
                    dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(typeQuotas);
                }
                dir.rootDir.CloneModificationTime(root);
                dir.rootDir.ClonePermissionStatus(root);
                AclFeature af = root.GetFeature(typeof(AclFeature));

                if (af != null)
                {
                    dir.rootDir.AddAclFeature(af);
                }
                // root dir supports having extended attributes according to POSIX
                XAttrFeature f = root.GetXAttrFeature();

                if (f != null)
                {
                    dir.rootDir.AddXAttrFeature(f);
                }
                dir.AddRootDirToEncryptionZone(f);
            }
Example #5
0
 /// <summary>Set the namespace, storagespace and typespace quota for a directory.</summary>
 /// <remarks>
 /// Set the namespace, storagespace and typespace quota for a directory.
 /// Note: This does not support ".inodes" relative path.
 /// </remarks>
 /// <exception cref="System.IO.IOException"/>
 internal static void SetQuota(FSDirectory fsd, string src, long nsQuota, long ssQuota
                               , StorageType type)
 {
     if (fsd.IsPermissionEnabled())
     {
         FSPermissionChecker pc = fsd.GetPermissionChecker();
         pc.CheckSuperuserPrivilege();
     }
     fsd.WriteLock();
     try
     {
         INodeDirectory changed = UnprotectedSetQuota(fsd, src, nsQuota, ssQuota, type);
         if (changed != null)
         {
             QuotaCounts q = changed.GetQuotaCounts();
             if (type == null)
             {
                 fsd.GetEditLog().LogSetQuota(src, q.GetNameSpace(), q.GetStorageSpace());
             }
             else
             {
                 fsd.GetEditLog().LogSetQuotaByStorageType(src, q.GetTypeSpaces().Get(type), type);
             }
         }
     }
     finally
     {
         fsd.WriteUnlock();
     }
 }
Example #6
0
        public override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps, byte
                                                      blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId)
        {
            DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();

            // we are computing the quota usage for a specific snapshot here, i.e., the
            // computation only includes files/directories that exist at the time of the
            // given snapshot
            if (sf != null && lastSnapshotId != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                .CurrentStateId && !(useCache && IsQuotaSet()))
            {
                ReadOnlyList <INode> childrenList = GetChildrenList(lastSnapshotId);
                foreach (INode child in childrenList)
                {
                    byte childPolicyId = child.GetStoragePolicyIDForQuota(blockStoragePolicyId);
                    child.ComputeQuotaUsage(bsps, childPolicyId, counts, useCache, lastSnapshotId);
                }
                counts.AddNameSpace(1);
                return(counts);
            }
            // compute the quota usage in the scope of the current directory tree
            DirectoryWithQuotaFeature q = GetDirectoryWithQuotaFeature();

            if (useCache && q != null && q.IsQuotaSet())
            {
                // use the cached quota
                return(q.AddCurrentSpaceUsage(counts));
            }
            else
            {
                useCache = q != null && !q.IsQuotaSet() ? false : useCache;
                return(ComputeDirectoryQuotaUsage(bsps, blockStoragePolicyId, counts, useCache, lastSnapshotId
                                                  ));
            }
        }
Example #7
0
            public static FsImageProto.INodeSection.INodeDirectory.Builder BuildINodeDirectory
                (INodeDirectoryAttributes dir, FSImageFormatProtobuf.SaverContext state)
            {
                QuotaCounts quota = dir.GetQuotaCounts();

                FsImageProto.INodeSection.INodeDirectory.Builder b = FsImageProto.INodeSection.INodeDirectory
                                                                     .NewBuilder().SetModificationTime(dir.GetModificationTime()).SetNsQuota(quota.GetNameSpace
                                                                                                                                                 ()).SetDsQuota(quota.GetStorageSpace()).SetPermission(BuildPermissionStatus(dir,
                                                                                                                                                                                                                             state.GetStringMap()));
                if (quota.GetTypeSpaces().AnyGreaterOrEqual(0))
                {
                    b.SetTypeQuotas(BuildQuotaByStorageTypeEntries(quota));
                }
                AclFeature f = dir.GetAclFeature();

                if (f != null)
                {
                    b.SetAcl(BuildAclEntries(f, state.GetStringMap()));
                }
                XAttrFeature xAttrFeature = dir.GetXAttrFeature();

                if (xAttrFeature != null)
                {
                    b.SetXAttrs(BuildXAttrs(xAttrFeature, state.GetStringMap()));
                }
                return(b);
            }
Example #8
0
 /// <summary>Check and add namespace/storagespace/storagetype consumed to itself and the ancestors.
 ///     </summary>
 /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">if quote is violated.
 ///     </exception>
 internal virtual void AddSpaceConsumed2Parent(QuotaCounts counts, bool verify)
 {
     if (parent != null)
     {
         parent.AddSpaceConsumed(counts, verify);
     }
 }
            public CopyWithQuota(INodeDirectory dir)
                : base(dir)
            {
                Preconditions.CheckArgument(dir.IsQuotaSet());
                QuotaCounts q = dir.GetQuotaCounts();

                this.quota = new QuotaCounts.Builder().QuotaCount(q).Build();
            }
 public CopyWithQuota(byte[] name, PermissionStatus permissions, AclFeature aclFeature
                      , long modificationTime, long nsQuota, long dsQuota, EnumCounters <StorageType> typeQuotas
                      , XAttrFeature xAttrsFeature)
     : base(name, permissions, aclFeature, modificationTime, xAttrsFeature)
 {
     this.quota = new QuotaCounts.Builder().NameSpace(nsQuota).StorageSpace(dsQuota).TypeSpaces
                      (typeQuotas).Build();
 }
Example #11
0
        public QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps, QuotaCounts counts
                                             , bool useCache)
        {
            byte storagePolicyId = IsSymlink() ? BlockStoragePolicySuite.IdUnspecified : GetStoragePolicyID
                                       ();

            return(ComputeQuotaUsage(bsps, storagePolicyId, counts, useCache, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                     .CurrentStateId));
        }
Example #12
0
 /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
 private static void VerifyQuota(FSDirectory fsd, INodesInPath targetIIP, QuotaCounts
                                 deltas)
 {
     if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
     {
         // Do not check quota if editlog is still being processed
         return;
     }
     FSDirectory.VerifyQuota(targetIIP, targetIIP.Length() - 1, deltas, null);
 }
Example #13
0
 /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
 internal virtual void UpdateQuotasInSourceTree(BlockStoragePolicySuite bsps)
 {
     // update the quota usage in src tree
     if (isSrcInSnapshot)
     {
         // get the counts after rename
         QuotaCounts newSrcCounts = srcChild.ComputeQuotaUsage(bsps, new QuotaCounts.Builder
                                                                   ().Build(), false);
         newSrcCounts.Subtract(oldSrcCounts);
         srcParent.AddSpaceConsumed(newSrcCounts, false);
     }
 }
Example #14
0
            /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
            internal RenameOperation(FSDirectory fsd, string src, string dst, INodesInPath srcIIP
                                     , INodesInPath dstIIP)
            {
                this.fsd          = fsd;
                this.src          = src;
                this.dst          = dst;
                this.srcIIP       = srcIIP;
                this.dstIIP       = dstIIP;
                this.srcParentIIP = srcIIP.GetParentINodesInPath();
                this.dstParentIIP = dstIIP.GetParentINodesInPath();
                BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();

                srcChild     = this.srcIIP.GetLastINode();
                srcChildName = srcChild.GetLocalNameBytes();
                int srcLatestSnapshotId = srcIIP.GetLatestSnapshotId();

                isSrcInSnapshot     = srcChild.IsInLatestSnapshot(srcLatestSnapshotId);
                srcChildIsReference = srcChild.IsReference();
                srcParent           = this.srcIIP.GetINode(-2).AsDirectory();
                // Record the snapshot on srcChild. After the rename, before any new
                // snapshot is taken on the dst tree, changes will be recorded in the
                // latest snapshot of the src tree.
                if (isSrcInSnapshot)
                {
                    srcChild.RecordModification(srcLatestSnapshotId);
                }
                // check srcChild for reference
                srcRefDstSnapshot = srcChildIsReference ? srcChild.AsReference().GetDstSnapshotId
                                        () : Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId;
                oldSrcCounts = new QuotaCounts.Builder().Build();
                if (isSrcInSnapshot)
                {
                    INodeReference.WithName withName = srcParent.ReplaceChild4ReferenceWithName(srcChild
                                                                                                , srcLatestSnapshotId);
                    withCount   = (INodeReference.WithCount)withName.GetReferredINode();
                    srcChild    = withName;
                    this.srcIIP = INodesInPath.Replace(srcIIP, srcIIP.Length() - 1, srcChild);
                    // get the counts before rename
                    withCount.GetReferredINode().ComputeQuotaUsage(bsps, oldSrcCounts, true);
                }
                else
                {
                    if (srcChildIsReference)
                    {
                        // srcChild is reference but srcChild is not in latest snapshot
                        withCount = (INodeReference.WithCount)srcChild.AsReference().GetReferredINode();
                    }
                    else
                    {
                        withCount = null;
                    }
                }
            }
        public virtual void TestQuotaByStorageTypePersistenceInFsImage()
        {
            string MethodName   = GenericTestUtils.GetMethodName();
            Path   testDir      = new Path(dir, MethodName);
            Path   createdFile1 = new Path(testDir, "created_file1.data");

            dfs.Mkdirs(testDir);
            // set storage policy on testDir to ONESSD
            dfs.SetStoragePolicy(testDir, HdfsConstants.OnessdStoragePolicyName);
            // set quota by storage type on testDir
            long SsdQuota = Blocksize * 4;

            dfs.SetQuotaByStorageType(testDir, StorageType.Ssd, SsdQuota);
            INode testDirNode = fsdir.GetINode4Write(testDir.ToString());

            NUnit.Framework.Assert.IsTrue(testDirNode.IsDirectory());
            NUnit.Framework.Assert.IsTrue(testDirNode.IsQuotaSet());
            // Create file of size 2 * BLOCKSIZE under testDir
            long file1Len = Blocksize * 2;
            int  bufLen   = Blocksize / 16;

            DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
                                   , seed);
            // Verify SSD consumed before namenode restart
            long ssdConsumed = testDirNode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
                                   ().GetTypeSpaces().Get(StorageType.Ssd);

            NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
            // Restart the namenode with checkpoint to make sure fsImage is correct
            dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
            dfs.SaveNamespace();
            dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
            cluster.RestartNameNode(true);
            RefreshClusterState();
            INode testDirNodeAfterNNRestart = fsdir.GetINode4Write(testDir.ToString());

            NUnit.Framework.Assert.IsTrue(testDirNode.IsDirectory());
            NUnit.Framework.Assert.IsTrue(testDirNode.IsQuotaSet());
            QuotaCounts qc = testDirNodeAfterNNRestart.GetQuotaCounts();

            NUnit.Framework.Assert.AreEqual(SsdQuota, qc.GetTypeSpace(StorageType.Ssd));
            foreach (StorageType t in StorageType.GetTypesSupportingQuota())
            {
                if (t != StorageType.Ssd)
                {
                    NUnit.Framework.Assert.AreEqual(HdfsConstants.QuotaReset, qc.GetTypeSpace(t));
                }
            }
            long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.AsDirectory().GetDirectoryWithQuotaFeature
                                                 ().GetSpaceConsumed().GetTypeSpaces().Get(StorageType.Ssd);

            NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumedAfterNNRestart);
        }
Example #16
0
        /// <summary>
        /// Compute
        /// <see cref="Org.Apache.Hadoop.FS.ContentSummary"/>
        /// .
        /// </summary>
        public ContentSummary ComputeAndConvertContentSummary(ContentSummaryComputationContext
                                                              summary)
        {
            ContentCounts counts = ComputeContentSummary(summary).GetCounts();
            QuotaCounts   q      = GetQuotaCounts();

            return(new ContentSummary.Builder().Length(counts.GetLength()).FileCount(counts.GetFileCount
                                                                                         () + counts.GetSymlinkCount()).DirectoryCount(counts.GetDirectoryCount()).Quota(
                       q.GetNameSpace()).SpaceConsumed(counts.GetStoragespace()).SpaceQuota(q.GetStorageSpace
                                                                                                ()).TypeConsumed(counts.GetTypeSpaces()).TypeQuota(q.GetTypeSpaces().AsArray()).
                   Build());
        }
Example #17
0
 private QuotaCounts ComputeDirectoryQuotaUsage(BlockStoragePolicySuite bsps, byte
                                                blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId)
 {
     if (children != null)
     {
         foreach (INode child in children)
         {
             byte childPolicyId = child.GetStoragePolicyIDForQuota(blockStoragePolicyId);
             child.ComputeQuotaUsage(bsps, childPolicyId, counts, useCache, lastSnapshotId);
         }
     }
     return(ComputeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts));
 }
Example #18
0
        /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
        public override void AddSpaceConsumed(QuotaCounts counts, bool verify)
        {
            DirectoryWithQuotaFeature q = GetDirectoryWithQuotaFeature();

            if (q != null)
            {
                q.AddSpaceConsumed(this, counts, verify);
            }
            else
            {
                AddSpaceConsumed2Parent(counts, verify);
            }
        }
Example #19
0
        /// <summary>
        /// Delete a path from the name space
        /// Update the count at each ancestor directory with quota
        /// </summary>
        /// <param name="iip">the inodes resolved from the path</param>
        /// <param name="collectedBlocks">blocks collected from the deleted path</param>
        /// <param name="removedINodes">inodes that should be removed from inodeMap</param>
        /// <param name="mtime">the time the inode is removed</param>
        /// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns>
        private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
                                              collectedBlocks, IList <INode> removedINodes, long mtime)
        {
            System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
            // check if target node exists
            INode targetNode = iip.GetLastINode();

            if (targetNode == null)
            {
                return(-1);
            }
            // record modification
            int latestSnapshot = iip.GetLatestSnapshotId();

            targetNode.RecordModification(latestSnapshot);
            // Remove the node from the namespace
            long removed = fsd.RemoveLastINode(iip);

            if (removed == -1)
            {
                return(-1);
            }
            // set the parent's modification time
            INodeDirectory parent = targetNode.GetParent();

            parent.UpdateModificationTime(mtime, latestSnapshot);
            fsd.UpdateCountForDelete(targetNode, iip);
            if (removed == 0)
            {
                return(0);
            }
            // collect block and update quota
            if (!targetNode.IsInLatestSnapshot(latestSnapshot))
            {
                targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks
                                                   , removedINodes);
            }
            else
            {
                QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                                             .CurrentStateId, latestSnapshot, collectedBlocks, removedINodes);
                removed = counts.GetNameSpace();
                fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation());
            }
            if (NameNode.stateChangeLog.IsDebugEnabled())
            {
                NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath
                                                  () + " is removed");
            }
            return(removed);
        }
Example #20
0
        /// <summary>Test if the quota can be correctly updated for append</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestUpdateQuotaForAppend()
        {
            Path foo            = new Path(dir, "foo");
            Path bar            = new Path(foo, "bar");
            long currentFileLen = Blocksize;

            DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed);
            dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
            // append half of the block data, the previous file length is at block
            // boundary
            DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2);
            currentFileLen += (Blocksize / 2);
            INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();

            NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet());
            QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            long        ns    = quota.GetNameSpace();
            long        ds    = quota.GetStorageSpace();

            NUnit.Framework.Assert.AreEqual(2, ns);
            // foo and bar
            NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
            ContentSummary c = dfs.GetContentSummary(foo);

            NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
            // append another block, the previous file length is not at block boundary
            DFSTestUtil.AppendFile(dfs, bar, Blocksize);
            currentFileLen += Blocksize;
            quota           = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            ns              = quota.GetNameSpace();
            ds              = quota.GetStorageSpace();
            NUnit.Framework.Assert.AreEqual(2, ns);
            // foo and bar
            NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
            c = dfs.GetContentSummary(foo);
            NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
            // append several blocks
            DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8);
            currentFileLen += (Blocksize * 3 + Blocksize / 8);
            quota           = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            ns              = quota.GetNameSpace();
            ds              = quota.GetStorageSpace();
            NUnit.Framework.Assert.AreEqual(2, ns);
            // foo and bar
            NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
            c = dfs.GetContentSummary(foo);
            NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
        }
Example #21
0
 private static FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto.Builder BuildQuotaByStorageTypeEntries
     (QuotaCounts q)
 {
     FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto.Builder b = FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto
                                                                          .NewBuilder();
     foreach (StorageType t in StorageType.GetTypesSupportingQuota())
     {
         if (q.GetTypeSpace(t) >= 0)
         {
             FsImageProto.INodeSection.QuotaByStorageTypeEntryProto.Builder eb = FsImageProto.INodeSection.QuotaByStorageTypeEntryProto
                                                                                 .NewBuilder().SetStorageType(PBHelper.ConvertStorageType(t)).SetQuota(q.GetTypeSpace
                                                                                                                                                           (t));
             b.AddQuotas(eb);
         }
     }
     return(b);
 }
Example #22
0
            public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
                                                         collectedBlocks, IList <INode> removedINodes)
            {
                int snapshot = GetSelfSnapshot();

                if (RemoveReference(this) <= 0)
                {
                    GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
                }
                else
                {
                    int   prior    = GetPriorSnapshot(this);
                    INode referred = GetReferredINode().AsReference().GetReferredINode();
                    if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
                    {
                        if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
                            snapshot <= prior)
                        {
                            // the snapshot to be deleted has been deleted while traversing
                            // the src tree of the previous rename operation. This usually
                            // happens when rename's src and dst are under the same
                            // snapshottable directory. E.g., the following operation sequence:
                            // 1. create snapshot s1 on /test
                            // 2. rename /test/foo/bar to /test/foo2/bar
                            // 3. create snapshot s2 on /test
                            // 4. rename foo2 again
                            // 5. delete snapshot s2
                            return;
                        }
                        try
                        {
                            QuotaCounts counts = referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks
                                                                       , removedINodes);
                            INodeReference @ref = GetReferredINode().GetParentReference();
                            if (@ref != null)
                            {
                                @ref.AddSpaceConsumed(counts.Negation(), true);
                            }
                        }
                        catch (QuotaExceededException e)
                        {
                            Log.Error("should not exceed quota while snapshot deletion", e);
                        }
                    }
                }
            }
Example #23
0
            public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
                                                     , int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
                                                     )
            {
                // since WithName node resides in deleted list acting as a snapshot copy,
                // the parameter snapshot must be non-null
                Preconditions.CheckArgument(snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                            .CurrentStateId);
                // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
                // previous WithName instance
                if (prior == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
                {
                    prior = GetPriorSnapshot(this);
                }
                if (prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId &&
                    Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdIntegerComparator
                    .Compare(snapshot, prior) <= 0)
                {
                    return(new QuotaCounts.Builder().Build());
                }
                QuotaCounts counts = GetReferredINode().CleanSubtree(bsps, snapshot, prior, collectedBlocks
                                                                     , removedINodes);
                INodeReference @ref = GetReferredINode().GetParentReference();

                if (@ref != null)
                {
                    try
                    {
                        @ref.AddSpaceConsumed(counts.Negation(), true);
                    }
                    catch (QuotaExceededException)
                    {
                        Org.Mortbay.Log.Log.Warn("Should not have QuotaExceededException");
                    }
                }
                if (snapshot < lastSnapshotId)
                {
                    // for a WithName node, when we compute its quota usage, we only count
                    // in all the nodes existing at the time of the corresponding rename op.
                    // Thus if we are deleting a snapshot before/at the snapshot associated
                    // with lastSnapshotId, we do not need to update the quota upwards.
                    counts = new QuotaCounts.Builder().Build();
                }
                return(counts);
            }
Example #24
0
        public override bool Equals(object obj)
        {
            if (obj == this)
            {
                return(true);
            }
            else
            {
                if (obj == null || !(obj is QuotaCounts))
                {
                    return(false);
                }
            }
            QuotaCounts that = (QuotaCounts)obj;

            return(this.nsSsCounts.Equals(that.nsSsCounts) && this.tsCounts.Equals(that.tsCounts
                                                                                   ));
        }
Example #25
0
        /// <summary>
        /// Undo the rename operation for the dst tree, i.e., if the rename operation
        /// (with OVERWRITE option) removes a file/dir from the dst tree, add it back
        /// and delete possible record in the deleted list.
        /// </summary>
        /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
        public virtual void UndoRename4DstParent(BlockStoragePolicySuite bsps, INode deletedChild
                                                 , int latestSnapshotId)
        {
            DirectoryWithSnapshotFeature sf = GetDirectoryWithSnapshotFeature();

            Preconditions.CheckState(sf != null, "Directory does not have snapshot feature");
            bool removeDeletedChild = sf.GetDiffs().RemoveChild(Diff.ListType.Deleted, deletedChild
                                                                );
            int sid = removeDeletedChild ? Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                      .CurrentStateId : latestSnapshotId;
            bool added = AddChild(deletedChild, true, sid);

            // update quota usage if adding is successfully and the old child has not
            // been stored in deleted list before
            if (added && !removeDeletedChild)
            {
                QuotaCounts counts = deletedChild.ComputeQuotaUsage(bsps);
                AddSpaceConsumed(counts, false);
            }
        }
Example #26
0
        /// <summary>Test if the quota can be correctly updated for create file</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestQuotaUpdateWithFileCreate()
        {
            Path foo         = new Path(dir, "foo");
            Path createdFile = new Path(foo, "created_file.data");

            dfs.Mkdirs(foo);
            dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
            long fileLen = Blocksize * 2 + Blocksize / 2;

            DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication
                                   , seed);
            INode fnode = fsdir.GetINode4Write(foo.ToString());

            NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
            NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
            QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
                                  ();

            NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace());
            NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace());
        }
Example #27
0
        /// <summary>
        /// Test if the quota can be correctly updated when file length is updated
        /// through fsync
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestUpdateQuotaForFSync()
        {
            Path foo = new Path("/foo");
            Path bar = new Path(foo, "bar");

            DFSTestUtil.CreateFile(dfs, bar, Blocksize, Replication, 0L);
            dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
            FSDataOutputStream @out = dfs.Append(bar);

            @out.Write(new byte[Blocksize / 4]);
            ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
                                                                        .UpdateLength));
            INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
            QuotaCounts    quota   = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            long           ns      = quota.GetNameSpace();
            long           ds      = quota.GetStorageSpace();

            NUnit.Framework.Assert.AreEqual(2, ns);
            // foo and bar
            NUnit.Framework.Assert.AreEqual(Blocksize * 2 * Replication, ds);
            // file is under construction
            @out.Write(new byte[Blocksize / 4]);
            @out.Close();
            fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
            quota   = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            ns      = quota.GetNameSpace();
            ds      = quota.GetStorageSpace();
            NUnit.Framework.Assert.AreEqual(2, ns);
            NUnit.Framework.Assert.AreEqual((Blocksize + Blocksize / 2) * Replication, ds);
            // append another block
            DFSTestUtil.AppendFile(dfs, bar, Blocksize);
            quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
            ns    = quota.GetNameSpace();
            ds    = quota.GetStorageSpace();
            NUnit.Framework.Assert.AreEqual(2, ns);
            // foo and bar
            NUnit.Framework.Assert.AreEqual((Blocksize * 2 + Blocksize / 2) * Replication, ds
                                            );
        }
        /// <summary>
        /// Both traditional space quota and the storage type quota for SSD are set and
        /// not exceeded.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestQuotaByStorageTypeWithTraditionalQuota()
        {
            Path foo = new Path(dir, "foo");

            dfs.Mkdirs(foo);
            dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
            dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
            dfs.SetQuota(foo, long.MaxValue - 1, Replication * Blocksize * 10);
            INode fnode = fsdir.GetINode4Write(foo.ToString());

            NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
            NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
            Path createdFile = new Path(foo, "created_file.data");
            long fileLen     = Blocksize * 2 + Blocksize / 2;

            DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication
                                   , seed);
            QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
                                  ();

            NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace());
            NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace());
            dfs.Delete(createdFile, true);
            QuotaCounts cntAfterDelete = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
                                             ();

            NUnit.Framework.Assert.AreEqual(1, cntAfterDelete.GetNameSpace());
            NUnit.Framework.Assert.AreEqual(0, cntAfterDelete.GetStorageSpace());
            // Validate the computeQuotaUsage()
            QuotaCounts counts = new QuotaCounts.Builder().Build();

            fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
                                    );
            NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 1, counts
                                            .GetNameSpace());
            NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
                                            .GetStorageSpace());
        }
Example #29
0
 /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
 internal void AddSpaceConsumed(INodeDirectory dir, QuotaCounts counts, bool verify
                                )
 {
     if (dir.IsQuotaSet())
     {
         // The following steps are important:
         // check quotas in this inode and all ancestors before changing counts
         // so that no change is made if there is any quota violation.
         // (1) verify quota in this inode
         if (verify)
         {
             VerifyQuota(counts);
         }
         // (2) verify quota and then add count in ancestors
         dir.AddSpaceConsumed2Parent(counts, verify);
         // (3) add count in this inode
         AddSpaceConsumed2Cache(counts);
     }
     else
     {
         dir.AddSpaceConsumed2Parent(counts, verify);
     }
 }
Example #30
0
        internal virtual void SetQuota(BlockStoragePolicySuite bsps, long nsQuota, long ssQuota
                                       , StorageType type)
        {
            DirectoryWithQuotaFeature quota = GetDirectoryWithQuotaFeature();

            if (quota != null)
            {
                // already has quota; so set the quota to the new values
                if (type != null)
                {
                    quota.SetQuota(ssQuota, type);
                }
                else
                {
                    quota.SetQuota(nsQuota, ssQuota);
                }
                if (!IsQuotaSet() && !IsRoot())
                {
                    RemoveFeature(quota);
                }
            }
            else
            {
                QuotaCounts c = ComputeQuotaUsage(bsps);
                DirectoryWithQuotaFeature.Builder builder = new DirectoryWithQuotaFeature.Builder
                                                                ().NameSpaceQuota(nsQuota);
                if (type != null)
                {
                    builder.TypeQuota(type, ssQuota);
                }
                else
                {
                    builder.StorageSpaceQuota(ssQuota);
                }
                AddDirectoryWithQuotaFeature(builder.Build()).SetSpaceConsumed(c);
            }
        }