/// <summary>Remove an inode from parent's children list.</summary>
/// <remarks>
/// Remove an inode from parent's children list. The caller of this method
/// needs to make sure that parent is in the given snapshot "latest".
/// </remarks>
public virtual bool RemoveChild(INodeDirectory parent, INode child, int latestSnapshotId
)
{
// For a directory that is not a renamed node, if isInLatestSnapshot returns
// false, the directory is not in the latest snapshot, thus we do not need
// to record the removed child in any snapshot.
// For a directory that was moved/renamed, note that if the directory is in
// any of the previous snapshots, we will create a reference node for the
// directory while rename, and isInLatestSnapshot will return true in that
// scenario (if all previous snapshots have been deleted, isInLatestSnapshot
// still returns false). Thus if isInLatestSnapshot returns false, the
// directory node cannot be in any snapshot (not in current tree, nor in
// previous src tree). Thus we do not need to record the removed child in
// any snapshot.
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff
(latestSnapshotId, parent).diff;
Diff.UndoInfo <INode> undoInfo = diff.Delete(child);
bool removed = parent.RemoveChild(child);
if (!removed && undoInfo != null)
{
// remove failed, undo
diff.UndoDelete(child, undoInfo);
}
return(removed);
}
/// <summary>Add a dir-diff pair</summary>
internal virtual void AddDirDiff(INodeDirectory dir, byte[][] relativePath, DirectoryWithSnapshotFeature.ChildrenDiff
diff)
{
dirDiffMap[dir] = diff;
diffMap[dir] = relativePath;
// detect rename
foreach (INode created in diff.GetList(Diff.ListType.Created))
{
if (created.IsReference())
{
SnapshotDiffInfo.RenameEntry entry = GetEntry(created.GetId());
if (entry.GetTargetPath() == null)
{
entry.SetTarget(created, relativePath);
}
}
}
foreach (INode deleted in diff.GetList(Diff.ListType.Deleted))
{
if (deleted is INodeReference.WithName)
{
SnapshotDiffInfo.RenameEntry entry = GetEntry(deleted.GetId());
entry.SetSource(deleted, relativePath);
}
}
}
private DirectoryDiff(int snapshotId, INodeDirectory dir)
: base(snapshotId, null, null)
{
this.childrenSize = dir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId).Size();
this.diff = new DirectoryWithSnapshotFeature.ChildrenDiff();
}
/// <summary>Compute the difference between Snapshots.</summary>
/// <param name="fromSnapshot">
/// Start point of the diff computation. Null indicates
/// current tree.
/// </param>
/// <param name="toSnapshot">
/// End point of the diff computation. Null indicates current
/// tree.
/// </param>
/// <param name="diff">
/// Used to capture the changes happening to the children. Note
/// that the diff still represents (later_snapshot - earlier_snapshot)
/// although toSnapshot can be before fromSnapshot.
/// </param>
/// <param name="currentINode">
/// The
/// <see cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeDirectory"/>
/// this feature belongs to.
/// </param>
/// <returns>Whether changes happened between the startSnapshot and endSnaphsot.</returns>
internal virtual bool ComputeDiffBetweenSnapshots(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
fromSnapshot, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot toSnapshot
, DirectoryWithSnapshotFeature.ChildrenDiff diff, INodeDirectory currentINode)
{
int[] diffIndexPair = diffs.ChangedBetweenSnapshots(fromSnapshot, toSnapshot);
if (diffIndexPair == null)
{
return(false);
}
int earlierDiffIndex = diffIndexPair[0];
int laterDiffIndex = diffIndexPair[1];
bool dirMetadataChanged = false;
INodeDirectoryAttributes dirCopy = null;
IList <DirectoryWithSnapshotFeature.DirectoryDiff> difflist = diffs.AsList();
for (int i = earlierDiffIndex; i < laterDiffIndex; i++)
{
DirectoryWithSnapshotFeature.DirectoryDiff sdiff = difflist[i];
diff.CombinePosterior(sdiff.diff, null);
if (!dirMetadataChanged && sdiff.snapshotINode != null)
{
if (dirCopy == null)
{
dirCopy = sdiff.snapshotINode;
}
else
{
if (!dirCopy.MetadataEquals(sdiff.snapshotINode))
{
dirMetadataChanged = true;
}
}
}
}
if (!diff.IsEmpty() || dirMetadataChanged)
{
return(true);
}
else
{
if (dirCopy != null)
{
for (int i_1 = laterDiffIndex; i_1 < difflist.Count; i_1++)
{
if (!dirCopy.MetadataEquals(difflist[i_1].snapshotINode))
{
return(true);
}
}
return(!dirCopy.MetadataEquals(currentINode));
}
else
{
return(false);
}
}
}
/// <summary>Constructor used by FSImage loading</summary>
internal DirectoryDiff(int snapshotId, INodeDirectoryAttributes snapshotINode, DirectoryWithSnapshotFeature.DirectoryDiff
posteriorDiff, int childrenSize, IList <INode> createdList, IList <INode> deletedList
, bool isSnapshotRoot)
: base(snapshotId, snapshotINode, posteriorDiff)
{
this.childrenSize = childrenSize;
this.diff = new DirectoryWithSnapshotFeature.ChildrenDiff(createdList, deletedList
);
this.isSnapshotRoot = isSnapshotRoot;
}
/// <summary>Remove the given child in the created/deleted list, if there is any.</summary>
public virtual bool RemoveChild(Diff.ListType type, INode child)
{
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = AsList();
for (int i = diffList.Count - 1; i >= 0; i--)
{
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffList[i].diff;
if (diff.RemoveChild(type, child))
{
return(true);
}
}
return(false);
}
/// <summary>Add an inode into parent's children list.</summary>
/// <remarks>
/// Add an inode into parent's children list. The caller of this method needs
/// to make sure that parent is in the given snapshot "latest".
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
public virtual bool AddChild(INodeDirectory parent, INode inode, bool setModTime,
int latestSnapshotId)
{
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff
(latestSnapshotId, parent).diff;
int undoInfo = diff.Create(inode);
bool added = parent.AddChild(inode, setModTime, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
if (!added)
{
diff.UndoCreate(inode, undoInfo);
}
return(added);
}
/// <summary>
/// Find the corresponding snapshot whose deleted list contains the given
/// inode.
/// </summary>
/// <returns>
/// the id of the snapshot.
/// <see cref="Snapshot.NoSnapshotId"/>
/// if the
/// given inode is not in any of the snapshot.
/// </returns>
public virtual int FindSnapshotDeleted(INode child)
{
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = AsList();
for (int i = diffList.Count - 1; i >= 0; i--)
{
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffList[i].diff;
int d = diff.SearchIndex(Diff.ListType.Deleted, child.GetLocalNameBytes());
if (d >= 0 && diff.GetList(Diff.ListType.Deleted)[d] == child)
{
return(diffList[i].GetSnapshotId());
}
}
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId);
}
private IList <INode> InitChildren()
{
if (this.children == null)
{
DirectoryWithSnapshotFeature.ChildrenDiff combined = new DirectoryWithSnapshotFeature.ChildrenDiff
();
for (DirectoryWithSnapshotFeature.DirectoryDiff d = this._enclosing; d != null; d
= d.GetPosterior())
{
combined.CombinePosterior(d.diff, null);
}
this.children = combined.Apply2Current(ReadOnlyList.Util.AsList(currentDir.GetChildrenList
(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)));
}
return(this.children);
}
/// <summary>
/// Interpret the ChildrenDiff and generate a list of
/// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotDiffReport.DiffReportEntry"/>
/// .
/// </summary>
/// <param name="dirDiff">The ChildrenDiff.</param>
/// <param name="parentPath">The relative path of the parent.</param>
/// <param name="fromEarlier">
/// True indicates
/// <c>diff=later-earlier</c>
/// ,
/// False indicates
/// <c>diff=earlier-later</c>
/// </param>
/// <param name="renameMap">A map containing information about rename operations.</param>
/// <returns>
/// A list of
/// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotDiffReport.DiffReportEntry"/>
/// as the diff report.
/// </returns>
private IList <SnapshotDiffReport.DiffReportEntry> GenerateReport(DirectoryWithSnapshotFeature.ChildrenDiff
dirDiff, byte[][] parentPath, bool fromEarlier, IDictionary <long, SnapshotDiffInfo.RenameEntry
> renameMap)
{
IList <SnapshotDiffReport.DiffReportEntry> list = new AList <SnapshotDiffReport.DiffReportEntry
>();
IList <INode> created = dirDiff.GetList(Diff.ListType.Created);
IList <INode> deleted = dirDiff.GetList(Diff.ListType.Deleted);
byte[][] fullPath = new byte[parentPath.Length + 1][];
System.Array.Copy(parentPath, 0, fullPath, 0, parentPath.Length);
foreach (INode cnode in created)
{
SnapshotDiffInfo.RenameEntry entry = renameMap[cnode.GetId()];
if (entry == null || !entry.IsRename())
{
fullPath[fullPath.Length - 1] = cnode.GetLocalNameBytes();
list.AddItem(new SnapshotDiffReport.DiffReportEntry(fromEarlier ? SnapshotDiffReport.DiffType
.Create : SnapshotDiffReport.DiffType.Delete, fullPath));
}
}
foreach (INode dnode in deleted)
{
SnapshotDiffInfo.RenameEntry entry = renameMap[dnode.GetId()];
if (entry != null && entry.IsRename())
{
list.AddItem(new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename
, fromEarlier ? entry.GetSourcePath() : entry.GetTargetPath(), fromEarlier ? entry
.GetTargetPath() : entry.GetSourcePath()));
}
else
{
fullPath[fullPath.Length - 1] = dnode.GetLocalNameBytes();
list.AddItem(new SnapshotDiffReport.DiffReportEntry(fromEarlier ? SnapshotDiffReport.DiffType
.Delete : SnapshotDiffReport.DiffType.Create, fullPath));
}
}
return(list);
}
/// <summary>
/// Recursively compute the difference between snapshots under a given
/// directory/file.
/// </summary>
/// <param name="snapshotRoot">The directory where snapshots were taken.</param>
/// <param name="node">The directory/file under which the diff is computed.</param>
/// <param name="parentPath">
/// Relative path (corresponding to the snapshot root) of
/// the node's parent.
/// </param>
/// <param name="diffReport">data structure used to store the diff.</param>
private void ComputeDiffRecursively(INodeDirectory snapshotRoot, INode node, IList
<byte[]> parentPath, SnapshotDiffInfo diffReport)
{
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot earlierSnapshot = diffReport
.IsFromEarlier() ? diffReport.GetFrom() : diffReport.GetTo();
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot laterSnapshot = diffReport
.IsFromEarlier() ? diffReport.GetTo() : diffReport.GetFrom();
byte[][] relativePath = Sharpen.Collections.ToArray(parentPath, new byte[parentPath
.Count][]);
if (node.IsDirectory())
{
DirectoryWithSnapshotFeature.ChildrenDiff diff = new DirectoryWithSnapshotFeature.ChildrenDiff
();
INodeDirectory dir = node.AsDirectory();
DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature();
if (sf != null)
{
bool change = sf.ComputeDiffBetweenSnapshots(earlierSnapshot, laterSnapshot, diff
, dir);
if (change)
{
diffReport.AddDirDiff(dir, relativePath, diff);
}
}
ReadOnlyList <INode> children = dir.GetChildrenList(earlierSnapshot.GetId());
foreach (INode child in children)
{
byte[] name = child.GetLocalNameBytes();
bool toProcess = diff.SearchIndex(Diff.ListType.Deleted, name) < 0;
if (!toProcess && child is INodeReference.WithName)
{
byte[][] renameTargetPath = FindRenameTargetPath(snapshotRoot, (INodeReference.WithName
)child, laterSnapshot == null ? Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId : laterSnapshot.GetId());
if (renameTargetPath != null)
{
toProcess = true;
diffReport.SetRenameTarget(child.GetId(), renameTargetPath);
}
}
if (toProcess)
{
parentPath.AddItem(name);
ComputeDiffRecursively(snapshotRoot, child, parentPath, diffReport);
parentPath.Remove(parentPath.Count - 1);
}
}
}
else
{
if (node.IsFile() && node.AsFile().IsWithSnapshot())
{
INodeFile file = node.AsFile();
bool change = file.GetFileWithSnapshotFeature().ChangedBetweenSnapshots(file, earlierSnapshot
, laterSnapshot);
if (change)
{
diffReport.AddFileDiff(file, relativePath);
}
}
}
}
/// <summary>
/// Clean an inode while we move it from the deleted list of post to the
/// deleted list of prior.
/// </summary>
/// <param name="bsps">The block storage policy suite.</param>
/// <param name="inode">The inode to clean.</param>
/// <param name="post">The post snapshot.</param>
/// <param name="prior">The id of the prior snapshot.</param>
/// <param name="collectedBlocks">Used to collect blocks for later deletion.</param>
/// <returns>Quota usage update.</returns>
private static QuotaCounts CleanDeletedINode(BlockStoragePolicySuite bsps, INode
inode, int post, int prior, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode
> removedINodes)
{
QuotaCounts counts = new QuotaCounts.Builder().Build();
Deque <INode> queue = new ArrayDeque <INode>();
queue.AddLast(inode);
while (!queue.IsEmpty())
{
INode topNode = queue.PollFirst();
if (topNode is INodeReference.WithName)
{
INodeReference.WithName wn = (INodeReference.WithName)topNode;
if (wn.GetLastSnapshotId() >= post)
{
wn.CleanSubtree(bsps, post, prior, collectedBlocks, removedINodes);
}
}
else
{
// For DstReference node, since the node is not in the created list of
// prior, we should treat it as regular file/dir
if (topNode.IsFile() && topNode.AsFile().IsWithSnapshot())
{
INodeFile file = topNode.AsFile();
counts.Add(file.GetDiffs().DeleteSnapshotDiff(bsps, post, prior, file, collectedBlocks
, removedINodes));
}
else
{
if (topNode.IsDirectory())
{
INodeDirectory dir = topNode.AsDirectory();
DirectoryWithSnapshotFeature.ChildrenDiff priorChildrenDiff = null;
DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature();
if (sf != null)
{
// delete files/dirs created after prior. Note that these
// files/dirs, along with inode, were deleted right after post.
DirectoryWithSnapshotFeature.DirectoryDiff priorDiff = sf.GetDiffs().GetDiffById(
prior);
if (priorDiff != null && priorDiff.GetSnapshotId() == prior)
{
priorChildrenDiff = priorDiff.GetChildrenDiff();
counts.Add(priorChildrenDiff.DestroyCreatedList(bsps, dir, collectedBlocks, removedINodes
));
}
}
foreach (INode child in dir.GetChildrenList(prior))
{
if (priorChildrenDiff != null && priorChildrenDiff.Search(Diff.ListType.Deleted,
child.GetLocalNameBytes()) != null)
{
continue;
}
queue.AddLast(child);
}
}
}
}
}
return(counts);
}
