/// <summary>
        /// If some blocks at the end of the block list no longer belongs to
        /// any inode, collect them and update the block list.
        /// </summary>
        public virtual void CollectBlocksAndClear(BlockStoragePolicySuite bsps, INodeFile
                                                  file, INode.BlocksMapUpdateInfo info, IList <INode> removedINodes)
        {
            // check if everything is deleted.
            if (IsCurrentFileDeleted() && GetDiffs().AsList().IsEmpty())
            {
                file.DestroyAndCollectBlocks(bsps, info, removedINodes);
                return;
            }
            // find max file size.
            long     max;
            FileDiff diff = GetDiffs().GetLast();

            if (IsCurrentFileDeleted())
            {
                max = diff == null ? 0 : diff.GetFileSize();
            }
            else
            {
                max = file.ComputeFileSize();
            }
            // Collect blocks that should be deleted
            FileDiff last = diffs.GetLast();

            BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.GetBlocks();
            if (snapshotBlocks == null)
            {
                file.CollectBlocksBeyondMax(max, info);
            }
            else
            {
                file.CollectBlocksBeyondSnapshot(snapshotBlocks, info);
            }
        }
Example #2
0
            /// <exception cref="System.IO.IOException"/>
            private void SerializeFileDiffList(INodeFile file, OutputStream @out)
            {
                FileWithSnapshotFeature sf = file.GetFileWithSnapshotFeature();

                if (sf != null)
                {
                    IList <FileDiff> diffList = sf.GetDiffs().AsList();
                    FsImageProto.SnapshotDiffSection.DiffEntry entry = ((FsImageProto.SnapshotDiffSection.DiffEntry
                                                                         )FsImageProto.SnapshotDiffSection.DiffEntry.NewBuilder().SetInodeId(file.GetId()
                                                                                                                                             ).SetType(FsImageProto.SnapshotDiffSection.DiffEntry.Type.Filediff).SetNumOfDiff
                                                                            (diffList.Count).Build());
                    entry.WriteDelimitedTo(@out);
                    for (int i = diffList.Count - 1; i >= 0; i--)
                    {
                        FileDiff diff = diffList[i];
                        FsImageProto.SnapshotDiffSection.FileDiff.Builder fb = FsImageProto.SnapshotDiffSection.FileDiff
                                                                               .NewBuilder().SetSnapshotId(diff.GetSnapshotId()).SetFileSize(diff.GetFileSize()
                                                                                                                                             );
                        if (diff.GetBlocks() != null)
                        {
                            foreach (Block block in diff.GetBlocks())
                            {
                                fb.AddBlocks(PBHelper.Convert(block));
                            }
                        }
                        INodeFileAttributes copy = diff.snapshotINode;
                        if (copy != null)
                        {
                            fb.SetName(ByteString.CopyFrom(copy.GetLocalNameBytes())).SetSnapshotCopy(FSImageFormatPBINode.Saver.BuildINodeFile
                                                                                                          (copy, parent.GetSaverContext()));
                        }
                        ((FsImageProto.SnapshotDiffSection.FileDiff)fb.Build()).WriteDelimitedTo(@out);
                    }
                }
            }
Example #3
0
            /// <summary>Load FileDiff list for a file with snapshot feature</summary>
            /// <exception cref="System.IO.IOException"/>
            private void LoadFileDiffList(InputStream @in, INodeFile file, int size)
            {
                FileDiffList diffs = new FileDiffList();

                FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
                for (int i = 0; i < size; i++)
                {
                    FsImageProto.SnapshotDiffSection.FileDiff pbf = FsImageProto.SnapshotDiffSection.FileDiff
                                                                    .ParseDelimitedFrom(@in);
                    INodeFileAttributes copy = null;
                    if (pbf.HasSnapshotCopy())
                    {
                        FsImageProto.INodeSection.INodeFile fileInPb = pbf.GetSnapshotCopy();
                        PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(fileInPb
                                                                                                 .GetPermission(), state.GetStringTable());
                        AclFeature acl = null;
                        if (fileInPb.HasAcl())
                        {
                            int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries
                                                                           (fileInPb.GetAcl(), state.GetStringTable()));
                            acl = new AclFeature(entries);
                        }
                        XAttrFeature xAttrs = null;
                        if (fileInPb.HasXAttrs())
                        {
                            xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(fileInPb.GetXAttrs
                                                                                                 (), state.GetStringTable()));
                        }
                        copy = new INodeFileAttributes.SnapshotCopy(pbf.GetName().ToByteArray(), permission
                                                                    , acl, fileInPb.GetModificationTime(), fileInPb.GetAccessTime(), (short)fileInPb
                                                                    .GetReplication(), fileInPb.GetPreferredBlockSize(), unchecked ((byte)fileInPb.GetStoragePolicyID
                                                                                                                                        ()), xAttrs);
                    }
                    FileDiff diff = new FileDiff(pbf.GetSnapshotId(), copy, null, pbf.GetFileSize());
                    IList <HdfsProtos.BlockProto> bpl    = pbf.GetBlocksList();
                    BlockInfoContiguous[]         blocks = new BlockInfoContiguous[bpl.Count];
                    for (int j = 0; j < e; ++j)
                    {
                        Block blk = PBHelper.Convert(bpl[j]);
                        BlockInfoContiguous storedBlock = fsn.GetBlockManager().GetStoredBlock(blk);
                        if (storedBlock == null)
                        {
                            storedBlock = fsn.GetBlockManager().AddBlockCollection(new BlockInfoContiguous(blk
                                                                                                           , copy.GetFileReplication()), file);
                        }
                        blocks[j] = storedBlock;
                    }
                    if (blocks.Length > 0)
                    {
                        diff.SetBlocks(blocks);
                    }
                    diffs.AddFirst(diff);
                }
                file.AddSnapshotFeature(diffs);
            }
Example #4
0
        /// <exception cref="System.IO.IOException"/>
        private static FileDiff LoadFileDiff(FileDiff posterior, DataInput @in, FSImageFormat.Loader
                                             loader)
        {
            // 1. Read the id of the Snapshot root to identify the Snapshot
            Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = loader.GetSnapshot
                                                                                    (@in);
            // 2. Load file size
            long fileSize = @in.ReadLong();
            // 3. Load snapshotINode
            INodeFileAttributes snapshotINode = @in.ReadBoolean() ? loader.LoadINodeFileAttributes
                                                    (@in) : null;

            return(new FileDiff(snapshot.GetId(), snapshotINode, posterior, fileSize));
        }
        internal virtual bool ChangedBetweenSnapshots(INodeFile file, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                                      from, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot to)
        {
            int[] diffIndexPair = diffs.ChangedBetweenSnapshots(from, to);
            if (diffIndexPair == null)
            {
                return(false);
            }
            int earlierDiffIndex           = diffIndexPair[0];
            int laterDiffIndex             = diffIndexPair[1];
            IList <FileDiff> diffList      = diffs.AsList();
            long             earlierLength = diffList[earlierDiffIndex].GetFileSize();
            long             laterLength   = laterDiffIndex == diffList.Count ? file.ComputeFileSize(true,
                                                                                                     false) : diffList[laterDiffIndex].GetFileSize();

            if (earlierLength != laterLength)
            {
                // file length has been changed
                return(true);
            }
            INodeFileAttributes earlierAttr = null;

            // check the metadata
            for (int i = earlierDiffIndex; i < laterDiffIndex; i++)
            {
                FileDiff diff = diffList[i];
                if (diff.snapshotINode != null)
                {
                    earlierAttr = diff.snapshotINode;
                    break;
                }
            }
            if (earlierAttr == null)
            {
                // no meta-change at all, return false
                return(false);
            }
            INodeFileAttributes laterAttr = diffs.GetSnapshotINode(Math.Max(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
                                                                            .GetSnapshotId(from), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotId
                                                                                (to)), file);

            return(!earlierAttr.MetadataEquals(laterAttr));
        }
Example #6
0
        /// <exception cref="System.IO.IOException"/>
        public static FileDiffList LoadFileDiffList(DataInput @in, FSImageFormat.Loader loader
                                                    )
        {
            int size = @in.ReadInt();

            if (size == -1)
            {
                return(null);
            }
            else
            {
                FileDiffList diffs     = new FileDiffList();
                FileDiff     posterior = null;
                for (int i = 0; i < size; i++)
                {
                    FileDiff d = LoadFileDiff(posterior, @in, loader);
                    diffs.AddFirst(d);
                    posterior = d;
                }
                return(diffs);
            }
        }
        public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps
                                                               , INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
                                                               <INode> removedINodes)
        {
            long oldStoragespace                  = file.StoragespaceConsumed();
            byte storagePolicyID                  = file.GetStoragePolicyID();
            BlockStoragePolicy         bsp        = null;
            EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType
                                                                                          ));

            if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified)
            {
                bsp = bsps.GetPolicy(file.GetStoragePolicyID());
            }
            if (removed.snapshotINode != null)
            {
                short replication = removed.snapshotINode.GetFileReplication();
                short currentRepl = file.GetBlockReplication();
                if (currentRepl == 0)
                {
                    long oldFileSizeNoRep = file.ComputeFileSize(true, true);
                    oldStoragespace = oldFileSizeNoRep * replication;
                    if (bsp != null)
                    {
                        IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
                        foreach (StorageType t in oldTypeChosen)
                        {
                            if (t.SupportTypeQuota())
                            {
                                typeSpaces.Add(t, -oldFileSizeNoRep);
                            }
                        }
                    }
                }
                else
                {
                    if (replication > currentRepl)
                    {
                        long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication();
                        oldStoragespace = oldFileSizeNoRep * replication;
                        if (bsp != null)
                        {
                            IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
                            foreach (StorageType t in oldTypeChosen)
                            {
                                if (t.SupportTypeQuota())
                                {
                                    typeSpaces.Add(t, -oldFileSizeNoRep);
                                }
                            }
                            IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl);
                            foreach (StorageType t_1 in newTypeChosen)
                            {
                                if (t_1.SupportTypeQuota())
                                {
                                    typeSpaces.Add(t_1, oldFileSizeNoRep);
                                }
                            }
                        }
                    }
                }
                AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature();
                if (aclFeature != null)
                {
                    AclStorage.RemoveAclFeature(aclFeature);
                }
            }
            GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks,
                                                       removedINodes);
            long ssDelta = oldStoragespace - file.StoragespaceConsumed();

            return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build
                       ());
        }