/// <exception cref="System.IO.IOException"/> private void SerializeDirDiffList(INodeDirectory dir, IList <INodeReference> refList , OutputStream @out) { DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = sf.GetDiffs().AsList (); FsImageProto.SnapshotDiffSection.DiffEntry entry = ((FsImageProto.SnapshotDiffSection.DiffEntry )FsImageProto.SnapshotDiffSection.DiffEntry.NewBuilder().SetInodeId(dir.GetId()) .SetType(FsImageProto.SnapshotDiffSection.DiffEntry.Type.Directorydiff).SetNumOfDiff (diffList.Count).Build()); entry.WriteDelimitedTo(@out); for (int i = diffList.Count - 1; i >= 0; i--) { // reverse order! DirectoryWithSnapshotFeature.DirectoryDiff diff = diffList[i]; FsImageProto.SnapshotDiffSection.DirectoryDiff.Builder db = FsImageProto.SnapshotDiffSection.DirectoryDiff .NewBuilder().SetSnapshotId(diff.GetSnapshotId()).SetChildrenSize(diff.GetChildrenSize ()).SetIsSnapshotRoot(diff.IsSnapshotRoot()); INodeDirectoryAttributes copy = diff.snapshotINode; if (!diff.IsSnapshotRoot() && copy != null) { db.SetName(ByteString.CopyFrom(copy.GetLocalNameBytes())).SetSnapshotCopy(FSImageFormatPBINode.Saver.BuildINodeDirectory (copy, parent.GetSaverContext())); } // process created list and deleted list IList <INode> created = diff.GetChildrenDiff().GetList(Diff.ListType.Created); db.SetCreatedListSize(created.Count); IList <INode> deleted = diff.GetChildrenDiff().GetList(Diff.ListType.Deleted); foreach (INode d in deleted) { if (d.IsReference()) { refList.AddItem(d.AsReference()); db.AddDeletedINodeRef(refList.Count - 1); } else { db.AddDeletedINode(d.GetId()); } } ((FsImageProto.SnapshotDiffSection.DirectoryDiff)db.Build()).WriteDelimitedTo(@out ); SaveCreatedList(created, @out); } } }
/// <summary> /// Clean an inode while we move it from the deleted list of post to the /// deleted list of prior. /// </summary> /// <param name="bsps">The block storage policy suite.</param> /// <param name="inode">The inode to clean.</param> /// <param name="post">The post snapshot.</param> /// <param name="prior">The id of the prior snapshot.</param> /// <param name="collectedBlocks">Used to collect blocks for later deletion.</param> /// <returns>Quota usage update.</returns> private static QuotaCounts CleanDeletedINode(BlockStoragePolicySuite bsps, INode inode, int post, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode > removedINodes) { QuotaCounts counts = new QuotaCounts.Builder().Build(); Deque <INode> queue = new ArrayDeque <INode>(); queue.AddLast(inode); while (!queue.IsEmpty()) { INode topNode = queue.PollFirst(); if (topNode is INodeReference.WithName) { INodeReference.WithName wn = (INodeReference.WithName)topNode; if (wn.GetLastSnapshotId() >= post) { wn.CleanSubtree(bsps, post, prior, collectedBlocks, removedINodes); } } else { // For DstReference node, since the node is not in the created list of // prior, we should treat it as regular file/dir if (topNode.IsFile() && topNode.AsFile().IsWithSnapshot()) { INodeFile file = topNode.AsFile(); counts.Add(file.GetDiffs().DeleteSnapshotDiff(bsps, post, prior, file, collectedBlocks , removedINodes)); } else { if (topNode.IsDirectory()) { INodeDirectory dir = topNode.AsDirectory(); DirectoryWithSnapshotFeature.ChildrenDiff priorChildrenDiff = null; DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { // delete files/dirs created after prior. Note that these // files/dirs, along with inode, were deleted right after post. DirectoryWithSnapshotFeature.DirectoryDiff priorDiff = sf.GetDiffs().GetDiffById( prior); if (priorDiff != null && priorDiff.GetSnapshotId() == prior) { priorChildrenDiff = priorDiff.GetChildrenDiff(); counts.Add(priorChildrenDiff.DestroyCreatedList(bsps, dir, collectedBlocks, removedINodes )); } } foreach (INode child in dir.GetChildrenList(prior)) { if (priorChildrenDiff != null && priorChildrenDiff.Search(Diff.ListType.Deleted, child.GetLocalNameBytes()) != null) { continue; } queue.AddLast(child); } } } } } return(counts); }
/// <summary> /// Recursively compute the difference between snapshots under a given /// directory/file. /// </summary> /// <param name="snapshotRoot">The directory where snapshots were taken.</param> /// <param name="node">The directory/file under which the diff is computed.</param> /// <param name="parentPath"> /// Relative path (corresponding to the snapshot root) of /// the node's parent. /// </param> /// <param name="diffReport">data structure used to store the diff.</param> private void ComputeDiffRecursively(INodeDirectory snapshotRoot, INode node, IList <byte[]> parentPath, SnapshotDiffInfo diffReport) { Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot earlierSnapshot = diffReport .IsFromEarlier() ? diffReport.GetFrom() : diffReport.GetTo(); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot laterSnapshot = diffReport .IsFromEarlier() ? diffReport.GetTo() : diffReport.GetFrom(); byte[][] relativePath = Sharpen.Collections.ToArray(parentPath, new byte[parentPath .Count][]); if (node.IsDirectory()) { DirectoryWithSnapshotFeature.ChildrenDiff diff = new DirectoryWithSnapshotFeature.ChildrenDiff (); INodeDirectory dir = node.AsDirectory(); DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { bool change = sf.ComputeDiffBetweenSnapshots(earlierSnapshot, laterSnapshot, diff , dir); if (change) { diffReport.AddDirDiff(dir, relativePath, diff); } } ReadOnlyList <INode> children = dir.GetChildrenList(earlierSnapshot.GetId()); foreach (INode child in children) { byte[] name = child.GetLocalNameBytes(); bool toProcess = diff.SearchIndex(Diff.ListType.Deleted, name) < 0; if (!toProcess && child is INodeReference.WithName) { byte[][] renameTargetPath = FindRenameTargetPath(snapshotRoot, (INodeReference.WithName )child, laterSnapshot == null ? Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId : laterSnapshot.GetId()); if (renameTargetPath != null) { toProcess = true; diffReport.SetRenameTarget(child.GetId(), renameTargetPath); } } if (toProcess) { parentPath.AddItem(name); ComputeDiffRecursively(snapshotRoot, child, parentPath, diffReport); parentPath.Remove(parentPath.Count - 1); } } } else { if (node.IsFile() && node.AsFile().IsWithSnapshot()) { INodeFile file = node.AsFile(); bool change = file.GetFileWithSnapshotFeature().ChangedBetweenSnapshots(file, earlierSnapshot , laterSnapshot); if (change) { diffReport.AddFileDiff(file, relativePath); } } } }
/// <summary>Destroy a subtree under a DstReference node.</summary> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> public static void DestroyDstSubtree(BlockStoragePolicySuite bsps, INode inode, int snapshot, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes ) { Preconditions.CheckArgument(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .NoSnapshotId); if (inode.IsReference()) { if (inode is INodeReference.WithName && snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId) { // this inode has been renamed before the deletion of the DstReference // subtree inode.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes); } else { // for DstReference node, continue this process to its subtree DestroyDstSubtree(bsps, inode.AsReference().GetReferredINode(), snapshot, prior, collectedBlocks, removedINodes); } } else { if (inode.IsFile()) { inode.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes); } else { if (inode.IsDirectory()) { IDictionary <INode, INode> excludedNodes = null; INodeDirectory dir = inode.AsDirectory(); DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { DirectoryWithSnapshotFeature.DirectoryDiffList diffList = sf.GetDiffs(); DirectoryWithSnapshotFeature.DirectoryDiff priorDiff = diffList.GetDiffById(prior ); if (priorDiff != null && priorDiff.GetSnapshotId() == prior) { IList <INode> dList = priorDiff.diff.GetList(Diff.ListType.Deleted); excludedNodes = CloneDiffList(dList); } if (snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId) { diffList.DeleteSnapshotDiff(bsps, snapshot, prior, dir, collectedBlocks, removedINodes ); } priorDiff = diffList.GetDiffById(prior); if (priorDiff != null && priorDiff.GetSnapshotId() == prior) { priorDiff.diff.DestroyCreatedList(bsps, dir, collectedBlocks, removedINodes); } } foreach (INode child in inode.AsDirectory().GetChildrenList(prior)) { if (excludedNodes != null && excludedNodes.Contains(child)) { continue; } DestroyDstSubtree(bsps, child, snapshot, prior, collectedBlocks, removedINodes); } } } } }