/// <summary>Remove an inode from parent's children list.</summary> /// <remarks> /// Remove an inode from parent's children list. The caller of this method /// needs to make sure that parent is in the given snapshot "latest". /// </remarks> public virtual bool RemoveChild(INodeDirectory parent, INode child, int latestSnapshotId ) { // For a directory that is not a renamed node, if isInLatestSnapshot returns // false, the directory is not in the latest snapshot, thus we do not need // to record the removed child in any snapshot. // For a directory that was moved/renamed, note that if the directory is in // any of the previous snapshots, we will create a reference node for the // directory while rename, and isInLatestSnapshot will return true in that // scenario (if all previous snapshots have been deleted, isInLatestSnapshot // still returns false). Thus if isInLatestSnapshot returns false, the // directory node cannot be in any snapshot (not in current tree, nor in // previous src tree). Thus we do not need to record the removed child in // any snapshot. DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff (latestSnapshotId, parent).diff; Diff.UndoInfo <INode> undoInfo = diff.Delete(child); bool removed = parent.RemoveChild(child); if (!removed && undoInfo != null) { // remove failed, undo diff.UndoDelete(child, undoInfo); } return(removed); }
/// <summary>Add a dir-diff pair</summary> internal virtual void AddDirDiff(INodeDirectory dir, byte[][] relativePath, DirectoryWithSnapshotFeature.ChildrenDiff diff) { dirDiffMap[dir] = diff; diffMap[dir] = relativePath; // detect rename foreach (INode created in diff.GetList(Diff.ListType.Created)) { if (created.IsReference()) { SnapshotDiffInfo.RenameEntry entry = GetEntry(created.GetId()); if (entry.GetTargetPath() == null) { entry.SetTarget(created, relativePath); } } } foreach (INode deleted in diff.GetList(Diff.ListType.Deleted)) { if (deleted is INodeReference.WithName) { SnapshotDiffInfo.RenameEntry entry = GetEntry(deleted.GetId()); entry.SetSource(deleted, relativePath); } } }
private DirectoryDiff(int snapshotId, INodeDirectory dir) : base(snapshotId, null, null) { this.childrenSize = dir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId).Size(); this.diff = new DirectoryWithSnapshotFeature.ChildrenDiff(); }
/// <summary>Compute the difference between Snapshots.</summary> /// <param name="fromSnapshot"> /// Start point of the diff computation. Null indicates /// current tree. /// </param> /// <param name="toSnapshot"> /// End point of the diff computation. Null indicates current /// tree. /// </param> /// <param name="diff"> /// Used to capture the changes happening to the children. Note /// that the diff still represents (later_snapshot - earlier_snapshot) /// although toSnapshot can be before fromSnapshot. /// </param> /// <param name="currentINode"> /// The /// <see cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeDirectory"/> /// this feature belongs to. /// </param> /// <returns>Whether changes happened between the startSnapshot and endSnaphsot.</returns> internal virtual bool ComputeDiffBetweenSnapshots(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot fromSnapshot, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot toSnapshot , DirectoryWithSnapshotFeature.ChildrenDiff diff, INodeDirectory currentINode) { int[] diffIndexPair = diffs.ChangedBetweenSnapshots(fromSnapshot, toSnapshot); if (diffIndexPair == null) { return(false); } int earlierDiffIndex = diffIndexPair[0]; int laterDiffIndex = diffIndexPair[1]; bool dirMetadataChanged = false; INodeDirectoryAttributes dirCopy = null; IList <DirectoryWithSnapshotFeature.DirectoryDiff> difflist = diffs.AsList(); for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { DirectoryWithSnapshotFeature.DirectoryDiff sdiff = difflist[i]; diff.CombinePosterior(sdiff.diff, null); if (!dirMetadataChanged && sdiff.snapshotINode != null) { if (dirCopy == null) { dirCopy = sdiff.snapshotINode; } else { if (!dirCopy.MetadataEquals(sdiff.snapshotINode)) { dirMetadataChanged = true; } } } } if (!diff.IsEmpty() || dirMetadataChanged) { return(true); } else { if (dirCopy != null) { for (int i_1 = laterDiffIndex; i_1 < difflist.Count; i_1++) { if (!dirCopy.MetadataEquals(difflist[i_1].snapshotINode)) { return(true); } } return(!dirCopy.MetadataEquals(currentINode)); } else { return(false); } } }
/// <summary>Constructor used by FSImage loading</summary> internal DirectoryDiff(int snapshotId, INodeDirectoryAttributes snapshotINode, DirectoryWithSnapshotFeature.DirectoryDiff posteriorDiff, int childrenSize, IList <INode> createdList, IList <INode> deletedList , bool isSnapshotRoot) : base(snapshotId, snapshotINode, posteriorDiff) { this.childrenSize = childrenSize; this.diff = new DirectoryWithSnapshotFeature.ChildrenDiff(createdList, deletedList ); this.isSnapshotRoot = isSnapshotRoot; }
/// <summary>Remove the given child in the created/deleted list, if there is any.</summary> public virtual bool RemoveChild(Diff.ListType type, INode child) { IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = AsList(); for (int i = diffList.Count - 1; i >= 0; i--) { DirectoryWithSnapshotFeature.ChildrenDiff diff = diffList[i].diff; if (diff.RemoveChild(type, child)) { return(true); } } return(false); }
/// <summary>Add an inode into parent's children list.</summary> /// <remarks> /// Add an inode into parent's children list. The caller of this method needs /// to make sure that parent is in the given snapshot "latest". /// </remarks> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> public virtual bool AddChild(INodeDirectory parent, INode inode, bool setModTime, int latestSnapshotId) { DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff (latestSnapshotId, parent).diff; int undoInfo = diff.Create(inode); bool added = parent.AddChild(inode, setModTime, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId); if (!added) { diff.UndoCreate(inode, undoInfo); } return(added); }
/// <summary> /// Find the corresponding snapshot whose deleted list contains the given /// inode. /// </summary> /// <returns> /// the id of the snapshot. /// <see cref="Snapshot.NoSnapshotId"/> /// if the /// given inode is not in any of the snapshot. /// </returns> public virtual int FindSnapshotDeleted(INode child) { IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = AsList(); for (int i = diffList.Count - 1; i >= 0; i--) { DirectoryWithSnapshotFeature.ChildrenDiff diff = diffList[i].diff; int d = diff.SearchIndex(Diff.ListType.Deleted, child.GetLocalNameBytes()); if (d >= 0 && diff.GetList(Diff.ListType.Deleted)[d] == child) { return(diffList[i].GetSnapshotId()); } } return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId); }
private IList <INode> InitChildren() { if (this.children == null) { DirectoryWithSnapshotFeature.ChildrenDiff combined = new DirectoryWithSnapshotFeature.ChildrenDiff (); for (DirectoryWithSnapshotFeature.DirectoryDiff d = this._enclosing; d != null; d = d.GetPosterior()) { combined.CombinePosterior(d.diff, null); } this.children = combined.Apply2Current(ReadOnlyList.Util.AsList(currentDir.GetChildrenList (Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId))); } return(this.children); }
/// <summary> /// Interpret the ChildrenDiff and generate a list of /// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotDiffReport.DiffReportEntry"/> /// . /// </summary> /// <param name="dirDiff">The ChildrenDiff.</param> /// <param name="parentPath">The relative path of the parent.</param> /// <param name="fromEarlier"> /// True indicates /// <c>diff=later-earlier</c> /// , /// False indicates /// <c>diff=earlier-later</c> /// </param> /// <param name="renameMap">A map containing information about rename operations.</param> /// <returns> /// A list of /// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotDiffReport.DiffReportEntry"/> /// as the diff report. /// </returns> private IList <SnapshotDiffReport.DiffReportEntry> GenerateReport(DirectoryWithSnapshotFeature.ChildrenDiff dirDiff, byte[][] parentPath, bool fromEarlier, IDictionary <long, SnapshotDiffInfo.RenameEntry > renameMap) { IList <SnapshotDiffReport.DiffReportEntry> list = new AList <SnapshotDiffReport.DiffReportEntry >(); IList <INode> created = dirDiff.GetList(Diff.ListType.Created); IList <INode> deleted = dirDiff.GetList(Diff.ListType.Deleted); byte[][] fullPath = new byte[parentPath.Length + 1][]; System.Array.Copy(parentPath, 0, fullPath, 0, parentPath.Length); foreach (INode cnode in created) { SnapshotDiffInfo.RenameEntry entry = renameMap[cnode.GetId()]; if (entry == null || !entry.IsRename()) { fullPath[fullPath.Length - 1] = cnode.GetLocalNameBytes(); list.AddItem(new SnapshotDiffReport.DiffReportEntry(fromEarlier ? SnapshotDiffReport.DiffType .Create : SnapshotDiffReport.DiffType.Delete, fullPath)); } } foreach (INode dnode in deleted) { SnapshotDiffInfo.RenameEntry entry = renameMap[dnode.GetId()]; if (entry != null && entry.IsRename()) { list.AddItem(new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename , fromEarlier ? entry.GetSourcePath() : entry.GetTargetPath(), fromEarlier ? entry .GetTargetPath() : entry.GetSourcePath())); } else { fullPath[fullPath.Length - 1] = dnode.GetLocalNameBytes(); list.AddItem(new SnapshotDiffReport.DiffReportEntry(fromEarlier ? SnapshotDiffReport.DiffType .Delete : SnapshotDiffReport.DiffType.Create, fullPath)); } } return(list); }
/// <summary> /// Recursively compute the difference between snapshots under a given /// directory/file. /// </summary> /// <param name="snapshotRoot">The directory where snapshots were taken.</param> /// <param name="node">The directory/file under which the diff is computed.</param> /// <param name="parentPath"> /// Relative path (corresponding to the snapshot root) of /// the node's parent. /// </param> /// <param name="diffReport">data structure used to store the diff.</param> private void ComputeDiffRecursively(INodeDirectory snapshotRoot, INode node, IList <byte[]> parentPath, SnapshotDiffInfo diffReport) { Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot earlierSnapshot = diffReport .IsFromEarlier() ? diffReport.GetFrom() : diffReport.GetTo(); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot laterSnapshot = diffReport .IsFromEarlier() ? diffReport.GetTo() : diffReport.GetFrom(); byte[][] relativePath = Sharpen.Collections.ToArray(parentPath, new byte[parentPath .Count][]); if (node.IsDirectory()) { DirectoryWithSnapshotFeature.ChildrenDiff diff = new DirectoryWithSnapshotFeature.ChildrenDiff (); INodeDirectory dir = node.AsDirectory(); DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { bool change = sf.ComputeDiffBetweenSnapshots(earlierSnapshot, laterSnapshot, diff , dir); if (change) { diffReport.AddDirDiff(dir, relativePath, diff); } } ReadOnlyList <INode> children = dir.GetChildrenList(earlierSnapshot.GetId()); foreach (INode child in children) { byte[] name = child.GetLocalNameBytes(); bool toProcess = diff.SearchIndex(Diff.ListType.Deleted, name) < 0; if (!toProcess && child is INodeReference.WithName) { byte[][] renameTargetPath = FindRenameTargetPath(snapshotRoot, (INodeReference.WithName )child, laterSnapshot == null ? Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId : laterSnapshot.GetId()); if (renameTargetPath != null) { toProcess = true; diffReport.SetRenameTarget(child.GetId(), renameTargetPath); } } if (toProcess) { parentPath.AddItem(name); ComputeDiffRecursively(snapshotRoot, child, parentPath, diffReport); parentPath.Remove(parentPath.Count - 1); } } } else { if (node.IsFile() && node.AsFile().IsWithSnapshot()) { INodeFile file = node.AsFile(); bool change = file.GetFileWithSnapshotFeature().ChangedBetweenSnapshots(file, earlierSnapshot , laterSnapshot); if (change) { diffReport.AddFileDiff(file, relativePath); } } } }
/// <summary> /// Clean an inode while we move it from the deleted list of post to the /// deleted list of prior. /// </summary> /// <param name="bsps">The block storage policy suite.</param> /// <param name="inode">The inode to clean.</param> /// <param name="post">The post snapshot.</param> /// <param name="prior">The id of the prior snapshot.</param> /// <param name="collectedBlocks">Used to collect blocks for later deletion.</param> /// <returns>Quota usage update.</returns> private static QuotaCounts CleanDeletedINode(BlockStoragePolicySuite bsps, INode inode, int post, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode > removedINodes) { QuotaCounts counts = new QuotaCounts.Builder().Build(); Deque <INode> queue = new ArrayDeque <INode>(); queue.AddLast(inode); while (!queue.IsEmpty()) { INode topNode = queue.PollFirst(); if (topNode is INodeReference.WithName) { INodeReference.WithName wn = (INodeReference.WithName)topNode; if (wn.GetLastSnapshotId() >= post) { wn.CleanSubtree(bsps, post, prior, collectedBlocks, removedINodes); } } else { // For DstReference node, since the node is not in the created list of // prior, we should treat it as regular file/dir if (topNode.IsFile() && topNode.AsFile().IsWithSnapshot()) { INodeFile file = topNode.AsFile(); counts.Add(file.GetDiffs().DeleteSnapshotDiff(bsps, post, prior, file, collectedBlocks , removedINodes)); } else { if (topNode.IsDirectory()) { INodeDirectory dir = topNode.AsDirectory(); DirectoryWithSnapshotFeature.ChildrenDiff priorChildrenDiff = null; DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature(); if (sf != null) { // delete files/dirs created after prior. Note that these // files/dirs, along with inode, were deleted right after post. DirectoryWithSnapshotFeature.DirectoryDiff priorDiff = sf.GetDiffs().GetDiffById( prior); if (priorDiff != null && priorDiff.GetSnapshotId() == prior) { priorChildrenDiff = priorDiff.GetChildrenDiff(); counts.Add(priorChildrenDiff.DestroyCreatedList(bsps, dir, collectedBlocks, removedINodes )); } } foreach (INode child in dir.GetChildrenList(prior)) { if (priorChildrenDiff != null && priorChildrenDiff.Search(Diff.ListType.Deleted, child.GetLocalNameBytes()) != null) { continue; } queue.AddLast(child); } } } } } return(counts); }