Exemplo n.º 1
0
        public long StoragespaceConsumedNoReplication()
        {
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf == null)
            {
                return(ComputeFileSize(true, true));
            }
            // Collect all distinct blocks
            long size = 0;
            ICollection <Block> allBlocks = new HashSet <Block>(Arrays.AsList(GetBlocks()));
            IList <FileDiff>    diffs     = sf.GetDiffs().AsList();

            foreach (FileDiff diff in diffs)
            {
                BlockInfoContiguous[] diffBlocks = diff.GetBlocks();
                if (diffBlocks != null)
                {
                    Sharpen.Collections.AddAll(allBlocks, Arrays.AsList(diffBlocks));
                }
            }
            foreach (Block block in allBlocks)
            {
                size += block.GetNumBytes();
            }
            // check if the last block is under construction
            BlockInfoContiguous lastBlock = GetLastBlock();

            if (lastBlock != null && lastBlock is BlockInfoContiguousUnderConstruction)
            {
                size += GetPreferredBlockSize() - lastBlock.GetNumBytes();
            }
            return(size);
        }
Exemplo n.º 2
0
        public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
                                                     collectedBlocks, IList <INode> removedINodes)
        {
            if (blocks != null && collectedBlocks != null)
            {
                foreach (BlockInfoContiguous blk in blocks)
                {
                    collectedBlocks.AddDeleteBlock(blk);
                    blk.SetBlockCollection(null);
                }
            }
            SetBlocks(null);
            if (GetAclFeature() != null)
            {
                AclStorage.RemoveAclFeature(GetAclFeature());
            }
            Clear();
            removedINodes.AddItem(this);
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf != null)
            {
                sf.GetDiffs().DestroyAndCollectSnapshotBlocks(collectedBlocks);
                sf.ClearDiffs();
            }
        }
Exemplo n.º 3
0
        public override string ToDetailString()
        {
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            return(base.ToDetailString() + (sf == null ? string.Empty : sf.GetDetailedString(
                                                )));
        }
Exemplo n.º 4
0
        public override QuotaCounts CleanSubtree(BlockStoragePolicySuite bsps, int snapshot
                                                 , int priorSnapshotId, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes
                                                 )
        {
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf != null)
            {
                return(sf.CleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, removedINodes
                                    ));
            }
            QuotaCounts counts = new QuotaCounts.Builder().Build();

            if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
            {
                if (priorSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId)
                {
                    // this only happens when deleting the current file and the file is not
                    // in any snapshot
                    ComputeQuotaUsage(bsps, counts, false);
                    DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
                }
                else
                {
                    // when deleting the current file and the file is in snapshot, we should
                    // clean the 0-sized block if the file is UC
                    FileUnderConstructionFeature uc = GetFileUnderConstructionFeature();
                    if (uc != null)
                    {
                        uc.CleanZeroSizeBlock(this, collectedBlocks);
                    }
                }
            }
            return(counts);
        }
Exemplo n.º 5
0
        /// <summary>Exclude blocks collected for deletion that belong to a snapshot.</summary>
        internal virtual void ExcludeSnapshotBlocks(int snapshotId, INode.BlocksMapUpdateInfo
                                                    collectedBlocks)
        {
            if (collectedBlocks == null || collectedBlocks.GetToDeleteList().IsEmpty())
            {
                return;
            }
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf == null)
            {
                return;
            }
            BlockInfoContiguous[] snapshotBlocks = GetDiffs().FindEarlierSnapshotBlocks(snapshotId
                                                                                        );
            if (snapshotBlocks == null)
            {
                return;
            }
            IList <Block> toDelete = collectedBlocks.GetToDeleteList();

            foreach (Block blk in snapshotBlocks)
            {
                if (toDelete.Contains(blk))
                {
                    collectedBlocks.RemoveDeleteBlock(blk);
                }
            }
        }
Exemplo n.º 6
0
        /* End of Under-Construction Feature */
        /* Start of Snapshot Feature */
        public virtual FileWithSnapshotFeature AddSnapshotFeature(FileDiffList diffs)
        {
            Preconditions.CheckState(!IsWithSnapshot(), "File is already with snapshot");
            FileWithSnapshotFeature sf = new FileWithSnapshotFeature(diffs);

            this.AddFeature(sf);
            return(sf);
        }
Exemplo n.º 7
0
        // This is the only place that needs to use the BlockStoragePolicySuite to
        // derive the intended storage type usage for quota by storage type
        public sealed override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps
                                                             , byte blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId
                                                             )
        {
            long  nsDelta = 1;
            long  ssDeltaNoReplication;
            short replication;
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf != null)
            {
                FileDiffList fileDiffList = sf.GetDiffs();
                int          last         = fileDiffList.GetLastSnapshotId();
                if (lastSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ||
                    last == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
                {
                    ssDeltaNoReplication = StoragespaceConsumedNoReplication();
                    replication          = GetBlockReplication();
                }
                else
                {
                    if (last < lastSnapshotId)
                    {
                        ssDeltaNoReplication = ComputeFileSize(true, false);
                        replication          = GetFileReplication();
                    }
                    else
                    {
                        int sid = fileDiffList.GetSnapshotById(lastSnapshotId);
                        ssDeltaNoReplication = StoragespaceConsumedNoReplication(sid);
                        replication          = GetReplication(sid);
                    }
                }
            }
            else
            {
                ssDeltaNoReplication = StoragespaceConsumedNoReplication();
                replication          = GetBlockReplication();
            }
            counts.AddNameSpace(nsDelta);
            counts.AddStorageSpace(ssDeltaNoReplication * replication);
            if (blockStoragePolicyId != BlockStoragePolicySuite.IdUnspecified)
            {
                BlockStoragePolicy  bsp          = bsps.GetPolicy(blockStoragePolicyId);
                IList <StorageType> storageTypes = bsp.ChooseStorageTypes(replication);
                foreach (StorageType t in storageTypes)
                {
                    if (!t.SupportTypeQuota())
                    {
                        continue;
                    }
                    counts.AddTypeSpace(t, ssDeltaNoReplication);
                }
            }
            return(counts);
        }
Exemplo n.º 8
0
        public virtual FileDiffList GetDiffs()
        {
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            if (sf != null)
            {
                return(sf.GetDiffs());
            }
            return(null);
        }
Exemplo n.º 9
0
        /// <returns>true if the block is contained in a snapshot or false otherwise.</returns>
        internal virtual bool IsBlockInLatestSnapshot(BlockInfoContiguous block)
        {
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            if (sf == null || sf.GetDiffs() == null)
            {
                return(false);
            }
            BlockInfoContiguous[] snapshotBlocks = GetDiffs().FindEarlierSnapshotBlocks(GetDiffs
                                                                                            ().GetLastSnapshotId());
            return(snapshotBlocks != null && Arrays.AsList(snapshotBlocks).Contains(block));
        }
Exemplo n.º 10
0
        public override INodeAttributes GetSnapshotINode(int snapshotId)
        {
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            if (sf != null)
            {
                return(sf.GetDiffs().GetSnapshotINode(snapshotId, this));
            }
            else
            {
                return(this);
            }
        }
Exemplo n.º 11
0
        /// <summary>
        /// Compute file size of the current file if the given snapshot is null;
        /// otherwise, get the file size from the given snapshot.
        /// </summary>
        public long ComputeFileSize(int snapshotId)
        {
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            if (snapshotId != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
                sf != null)
            {
                FileDiff d = sf.GetDiffs().GetDiffById(snapshotId);
                if (d != null)
                {
                    return(d.GetFileSize());
                }
            }
            return(ComputeFileSize(true, false));
        }
Exemplo n.º 12
0
 public virtual void RecordModification(int latestSnapshotId, bool withBlocks)
 {
     if (IsInLatestSnapshot(latestSnapshotId) && !ShouldRecordInSrcSnapshot(latestSnapshotId
                                                                            ))
     {
         // the file is in snapshot, create a snapshot feature if it does not have
         FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();
         if (sf == null)
         {
             sf = AddSnapshotFeature(null);
         }
         // record self in the diff list if necessary
         sf.GetDiffs().SaveSelf2Snapshot(latestSnapshotId, this, null, withBlocks);
     }
 }
Exemplo n.º 13
0
        public virtual short GetBlockReplication()
        {
            // BlockCollection
            short max = GetFileReplication(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                           .CurrentStateId);
            FileWithSnapshotFeature sf = this.GetFileWithSnapshotFeature();

            if (sf != null)
            {
                short maxInSnapshot = sf.GetMaxBlockRepInDiffs();
                if (sf.IsCurrentFileDeleted())
                {
                    return(maxInSnapshot);
                }
                max = maxInSnapshot > max ? maxInSnapshot : max;
            }
            return(max);
        }
Exemplo n.º 14
0
        public sealed override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext
                                                                                      summary)
        {
            ContentCounts           counts = summary.GetCounts();
            FileWithSnapshotFeature sf     = GetFileWithSnapshotFeature();
            long fileLen = 0;

            if (sf == null)
            {
                fileLen = ComputeFileSize();
                counts.AddContent(Content.File, 1);
            }
            else
            {
                FileDiffList diffs = sf.GetDiffs();
                int          n     = diffs.AsList().Count;
                counts.AddContent(Content.File, n);
                if (n > 0 && sf.IsCurrentFileDeleted())
                {
                    fileLen = diffs.GetLast().GetFileSize();
                }
                else
                {
                    fileLen = ComputeFileSize();
                }
            }
            counts.AddContent(Content.Length, fileLen);
            counts.AddContent(Content.Diskspace, StoragespaceConsumed());
            if (GetStoragePolicyID() != BlockStoragePolicySuite.IdUnspecified)
            {
                BlockStoragePolicy bsp = summary.GetBlockStoragePolicySuite().GetPolicy(GetStoragePolicyID
                                                                                            ());
                IList <StorageType> storageTypes = bsp.ChooseStorageTypes(GetFileReplication());
                foreach (StorageType t in storageTypes)
                {
                    if (!t.SupportTypeQuota())
                    {
                        continue;
                    }
                    counts.AddTypeSpace(t, fileLen);
                }
            }
            return(summary);
        }
Exemplo n.º 15
0
        /// <summary>compute the quota usage change for a truncate op</summary>
        /// <param name="newLength">the length for truncation</param>
        /// <returns>the quota usage delta (not considering replication factor)</returns>
        internal virtual long ComputeQuotaDeltaForTruncate(long newLength)
        {
            BlockInfoContiguous[] blocks = GetBlocks();
            if (blocks == null || blocks.Length == 0)
            {
                return(0);
            }
            int  n    = 0;
            long size = 0;

            for (; n < blocks.Length && newLength > size; n++)
            {
                size += blocks[n].GetNumBytes();
            }
            bool onBoundary   = size == newLength;
            long truncateSize = 0;

            for (int i = (onBoundary ? n : n - 1); i < blocks.Length; i++)
            {
                truncateSize += blocks[i].GetNumBytes();
            }
            FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();

            if (sf != null)
            {
                FileDiff diff = sf.GetDiffs().GetLast();
                BlockInfoContiguous[] sblocks = diff != null?diff.GetBlocks() : null;

                if (sblocks != null)
                {
                    for (int i_1 = (onBoundary ? n : n - 1); i_1 < blocks.Length && i_1 < sblocks.Length &&
                         blocks[i_1].Equals(sblocks[i_1]); i_1++)
                    {
                        truncateSize -= blocks[i_1].GetNumBytes();
                    }
                }
            }
            return(onBoundary ? -truncateSize : (GetPreferredBlockSize() - truncateSize));
        }