/// <summary>Load the snapshots section from fsimage.</summary>
/// <remarks>
/// Load the snapshots section from fsimage. Also add snapshottable feature
/// to snapshottable directories.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public void LoadSnapshotSection(InputStream @in)
{
SnapshotManager sm = fsn.GetSnapshotManager();
FsImageProto.SnapshotSection section = FsImageProto.SnapshotSection.ParseDelimitedFrom
(@in);
int snum = section.GetNumSnapshots();
sm.SetNumSnapshots(snum);
sm.SetSnapshotCounter(section.GetSnapshotCounter());
foreach (long sdirId in section.GetSnapshottableDirList())
{
INodeDirectory dir = fsDir.GetInode(sdirId).AsDirectory();
if (!dir.IsSnapshottable())
{
dir.AddSnapshottableFeature();
}
else
{
// dir is root, and admin set root to snapshottable before
dir.SetSnapshotQuota(DirectorySnapshottableFeature.SnapshotLimit);
}
sm.AddSnapshottable(dir);
}
LoadSnapshots(@in, snum);
}
/// <returns>
/// If there is no corresponding directory diff for the given
/// snapshot, this means that the current children list should be
/// returned for the snapshot. Otherwise we calculate the children list
/// for the snapshot and return it.
/// </returns>
public virtual ReadOnlyList <INode> GetChildrenList(INodeDirectory currentINode, int
snapshotId)
{
DirectoryWithSnapshotFeature.DirectoryDiff diff = diffs.GetDiffById(snapshotId);
return(diff != null?diff.GetChildrenList(currentINode) : currentINode.GetChildrenList
(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId));
}
/// <summary>Rename the given snapshot</summary>
/// <param name="oldSnapshotName">Old name of the snapshot</param>
/// <param name="newSnapshotName">New name of the snapshot</param>
/// <exception cref="System.IO.IOException">
/// Throw IOException when 1) the given path does not lead to an
/// existing snapshottable directory, and/or 2) the snapshot with the
/// old name does not exist for the directory, and/or 3) there exists
/// a snapshot with the new name for the directory
/// </exception>
public virtual void RenameSnapshot(INodesInPath iip, string snapshotRoot, string
oldSnapshotName, string newSnapshotName)
{
INodeDirectory srcRoot = GetSnapshottableRoot(iip);
srcRoot.RenameSnapshot(snapshotRoot, oldSnapshotName, newSnapshotName);
}
/// <summary>Add a snapshot.</summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
public virtual Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot AddSnapshot
(INodeDirectory snapshotRoot, int id, string name)
{
//check snapshot quota
int n = GetNumSnapshots();
if (n + 1 > snapshotQuota)
{
throw new SnapshotException("Failed to add snapshot: there are already " + n + " snapshot(s) and the snapshot quota is "
+ snapshotQuota);
}
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s = new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
(id, name, snapshotRoot);
byte[] nameBytes = s.GetRoot().GetLocalNameBytes();
int i = SearchSnapshot(nameBytes);
if (i >= 0)
{
throw new SnapshotException("Failed to add snapshot: there is already a " + "snapshot with the same name \""
+ Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.GetSnapshotName(s) +
"\".");
}
DirectoryWithSnapshotFeature.DirectoryDiff d = GetDiffs().AddDiff(id, snapshotRoot
);
d.SetSnapshotRoot(s.GetRoot());
snapshotsByNames.Add(-i - 1, s);
// set modification time
long now = Time.Now();
snapshotRoot.UpdateModificationTime(now, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
s.GetRoot().SetModificationTime(now, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
return(s);
}
/// <summary>Load a node stored in the created list from fsimage.</summary>
/// <param name="createdNodeName">The name of the created node.</param>
/// <param name="parent">The directory that the created list belongs to.</param>
/// <returns>The created node.</returns>
/// <exception cref="System.IO.IOException"/>
public static INode LoadCreated(byte[] createdNodeName, INodeDirectory parent)
{
// the INode in the created list should be a reference to another INode
// in posterior SnapshotDiffs or one of the current children
foreach (DirectoryWithSnapshotFeature.DirectoryDiff postDiff in parent.GetDiffs())
{
INode d = postDiff.GetChildrenDiff().Search(Diff.ListType.Deleted, createdNodeName
);
if (d != null)
{
return(d);
}
}
// else go to the next SnapshotDiff
// use the current child
INode currentChild = parent.GetChild(createdNodeName, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
if (currentChild == null)
{
throw new IOException("Cannot find an INode associated with the INode " + DFSUtil
.Bytes2String(createdNodeName) + " in created list while loading FSImage.");
}
return(currentChild);
}
public static SnapshottableDirectoryStatus.Bean ToBean(INodeDirectory d)
{
return(new SnapshottableDirectoryStatus.Bean(d.GetFullPathName(), d.GetDirectorySnapshottableFeature
().GetNumSnapshots(), d.GetDirectorySnapshottableFeature().GetSnapshotQuota(), d
.GetModificationTime(), short.ValueOf(Sharpen.Extensions.ToOctalString(d.GetFsPermissionShort
())), d.GetUserName(), d.GetGroupName()));
}
/// <summary>Set the given snapshottable directory to non-snapshottable.</summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException">if there are snapshots in the directory.
/// </exception>
/// <exception cref="System.IO.IOException"/>
public virtual void ResetSnapshottable(string path)
{
INodesInPath iip = fsdir.GetINodesInPath4Write(path);
INodeDirectory d = INodeDirectory.ValueOf(iip.GetLastINode(), path);
DirectorySnapshottableFeature sf = d.GetDirectorySnapshottableFeature();
if (sf == null)
{
// the directory is already non-snapshottable
return;
}
if (sf.GetNumSnapshots() > 0)
{
throw new SnapshotException("The directory " + path + " has snapshot(s). " + "Please redo the operation after removing all the snapshots."
);
}
if (d == fsdir.GetRoot())
{
d.SetSnapshotQuota(0);
}
else
{
d.RemoveSnapshottableFeature();
}
RemoveSnapshottable(d);
}
public virtual void DumpTreeRecursively(INodeDirectory snapshotRoot, PrintWriter
@out, StringBuilder prefix, int snapshot)
{
if (snapshot == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
@out.WriteLine();
@out.Write(prefix);
@out.Write("Snapshot of ");
string name = snapshotRoot.GetLocalName();
@out.Write(name.IsEmpty() ? "/" : name);
@out.Write(": quota=");
@out.Write(GetSnapshotQuota());
int n = 0;
foreach (DirectoryWithSnapshotFeature.DirectoryDiff diff in GetDiffs())
{
if (diff.IsSnapshotRoot())
{
n++;
}
}
Preconditions.CheckState(n == snapshotsByNames.Count, "#n=" + n + ", snapshotsByNames.size()="
+ snapshotsByNames.Count);
@out.Write(", #snapshot=");
@out.WriteLine(n);
INodeDirectory.DumpTreeRecursively(@out, prefix, new _IEnumerable_416(this));
}
}
public virtual INode GetChild(INodeDirectory currentINode, byte[] name, int snapshotId
)
{
DirectoryWithSnapshotFeature.DirectoryDiff diff = diffs.GetDiffById(snapshotId);
return(diff != null?diff.GetChild(name, true, currentINode) : currentINode.GetChild
(name, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId));
}
/// <summary>Remove an inode from parent's children list.</summary>
/// <remarks>
/// Remove an inode from parent's children list. The caller of this method
/// needs to make sure that parent is in the given snapshot "latest".
/// </remarks>
public virtual bool RemoveChild(INodeDirectory parent, INode child, int latestSnapshotId
)
{
// For a directory that is not a renamed node, if isInLatestSnapshot returns
// false, the directory is not in the latest snapshot, thus we do not need
// to record the removed child in any snapshot.
// For a directory that was moved/renamed, note that if the directory is in
// any of the previous snapshots, we will create a reference node for the
// directory while rename, and isInLatestSnapshot will return true in that
// scenario (if all previous snapshots have been deleted, isInLatestSnapshot
// still returns false). Thus if isInLatestSnapshot returns false, the
// directory node cannot be in any snapshot (not in current tree, nor in
// previous src tree). Thus we do not need to record the removed child in
// any snapshot.
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff
(latestSnapshotId, parent).diff;
Diff.UndoInfo <INode> undoInfo = diff.Delete(child);
bool removed = parent.RemoveChild(child);
if (!removed && undoInfo != null)
{
// remove failed, undo
diff.UndoDelete(child, undoInfo);
}
return(removed);
}
/// <summary>Add a dir-diff pair</summary>
internal virtual void AddDirDiff(INodeDirectory dir, byte[][] relativePath, DirectoryWithSnapshotFeature.ChildrenDiff
diff)
{
dirDiffMap[dir] = diff;
diffMap[dir] = relativePath;
// detect rename
foreach (INode created in diff.GetList(Diff.ListType.Created))
{
if (created.IsReference())
{
SnapshotDiffInfo.RenameEntry entry = GetEntry(created.GetId());
if (entry.GetTargetPath() == null)
{
entry.SetTarget(created, relativePath);
}
}
}
foreach (INode deleted in diff.GetList(Diff.ListType.Deleted))
{
if (deleted is INodeReference.WithName)
{
SnapshotDiffInfo.RenameEntry entry = GetEntry(deleted.GetId());
entry.SetSource(deleted, relativePath);
}
}
}
internal static void Modify(INode inode, IList <INode> current, Diff <byte[], INode
> diff)
{
int i = Diff.Search(current, inode.GetKey());
NUnit.Framework.Assert.IsTrue(i >= 0);
INodeDirectory oldinode = (INodeDirectory)current[i];
INodeDirectory newinode = new INodeDirectory(oldinode, false, oldinode.GetFeatures
());
newinode.SetModificationTime(oldinode.GetModificationTime() + 1);
current.Set(i, newinode);
if (diff != null)
{
//test undo with 1/UNDO_TEST_P probability
bool testUndo = Random.Next(UndoTestP) == 0;
string before = null;
if (testUndo)
{
before = diff.ToString();
}
Diff.UndoInfo <INode> undoInfo = diff.Modify(oldinode, newinode);
if (testUndo)
{
string after = diff.ToString();
//undo
diff.UndoModify(oldinode, newinode, undoInfo);
AssertDiff(before, diff);
//re-do
diff.Modify(oldinode, newinode);
AssertDiff(after, diff);
}
}
}
/// <summary>
/// Test
/// <see cref="Snapshot.IdComparator"/>
/// .
/// </summary>
public virtual void TestIdCmp()
{
PermissionStatus perm = PermissionStatus.CreateImmutable("user", "group", FsPermission
.CreateImmutable((short)0));
INodeDirectory snapshottable = new INodeDirectory(0, DFSUtil.String2Bytes("foo"),
perm, 0L);
snapshottable.AddSnapshottableFeature();
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot[] snapshots = new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
[] { new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot(1, "s1", snapshottable
), new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot(1, "s1", snapshottable
), new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot(2, "s2", snapshottable
), new Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot(2, "s2", snapshottable
) };
NUnit.Framework.Assert.AreEqual(0, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.IdComparator.Compare(null, null));
foreach (Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s in snapshots)
{
NUnit.Framework.Assert.IsTrue(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.IdComparator.Compare(null, s) > 0);
NUnit.Framework.Assert.IsTrue(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.IdComparator.Compare(s, null) < 0);
foreach (Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot t in snapshots)
{
int expected = string.CompareOrdinal(s.GetRoot().GetLocalName(), t.GetRoot().GetLocalName
());
int computed = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.IdComparator
.Compare(s, t);
NUnit.Framework.Assert.AreEqual(expected > 0, computed > 0);
NUnit.Framework.Assert.AreEqual(expected == 0, computed == 0);
NUnit.Framework.Assert.AreEqual(expected < 0, computed < 0);
}
}
}
public virtual ContentSummaryComputationContext ComputeContentSummary(BlockStoragePolicySuite
bsps, INodeDirectory snapshotRoot, ContentSummaryComputationContext summary)
{
snapshotRoot.ComputeContentSummary(summary);
summary.GetCounts().AddContent(Content.Snapshot, snapshotsByNames.Count);
summary.GetCounts().AddContent(Content.SnapshottableDirectory, 1);
return(summary);
}
internal SnapshotDiffInfo(INodeDirectory snapshotRoot, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
start, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot end)
{
Preconditions.CheckArgument(snapshotRoot.IsSnapshottable());
this.snapshotRoot = snapshotRoot;
this.from = start;
this.to = end;
}
/// <summary>Delete a snapshot for a snapshottable directory</summary>
/// <param name="snapshotName">Name of the snapshot to be deleted</param>
/// <param name="collectedBlocks">Used to collect information to update blocksMap</param>
/// <exception cref="System.IO.IOException"/>
public virtual void DeleteSnapshot(INodesInPath iip, string snapshotName, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
INodeDirectory srcRoot = GetSnapshottableRoot(iip);
srcRoot.RemoveSnapshot(fsdir.GetBlockStoragePolicySuite(), snapshotName, collectedBlocks
, removedINodes);
numSnapshots.GetAndDecrement();
}
private void AddToDeletedList(INode dnode, INodeDirectory parent)
{
dnode.SetParent(parent);
if (dnode.IsFile())
{
FSImageFormatPBINode.Loader.UpdateBlocksMap(dnode.AsFile(), fsn.GetBlockManager()
);
}
}
public virtual void TestClearQuota()
{
Path dir = new Path("/TestSnapshot");
hdfs.Mkdirs(dir);
hdfs.AllowSnapshot(dir);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet, HdfsConstants.QuotaDontSet);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaDontSet - 1, HdfsConstants.QuotaDontSet - 1
);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(0, dirNode.GetDiffs().AsList().Count);
// allow snapshot on dir and create snapshot s1
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s1");
// clear quota of dir
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
// dir should still be a snapshottable directory
dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(dirNode.IsSnapshottable());
NUnit.Framework.Assert.AreEqual(1, dirNode.GetDiffs().AsList().Count);
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
NUnit.Framework.Assert.AreEqual(dir, status[0].GetFullPath());
Path subDir = new Path(dir, "sub");
hdfs.Mkdirs(subDir);
hdfs.CreateSnapshot(dir, "s2");
Path file = new Path(subDir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
hdfs.SetQuota(dir, HdfsConstants.QuotaReset, HdfsConstants.QuotaReset);
INode subNode = fsdir.GetINode4Write(subDir.ToString());
NUnit.Framework.Assert.IsTrue(subNode.AsDirectory().IsWithSnapshot());
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = subNode.AsDirectory(
).GetDiffs().AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s2 = dirNode.GetSnapshot
(DFSUtil.String2Bytes("s2"));
NUnit.Framework.Assert.AreEqual(s2.GetId(), diffList[0].GetSnapshotId());
IList <INode> createdList = diffList[0].GetChildrenDiff().GetList(Diff.ListType.Created
);
NUnit.Framework.Assert.AreEqual(1, createdList.Count);
NUnit.Framework.Assert.AreSame(fsdir.GetINode4Write(file.ToString()), createdList
[0]);
}
public virtual void Clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode
, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes)
{
// destroy its diff list
foreach (DirectoryWithSnapshotFeature.DirectoryDiff diff in diffs)
{
diff.DestroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks, removedINodes
);
}
diffs.Clear();
}
/// <summary>
/// Find the source root directory where the snapshot will be taken
/// for a given path.
/// </summary>
/// <returns>Snapshottable directory.</returns>
/// <exception cref="System.IO.IOException">
/// Throw IOException when the given path does not lead to an
/// existing snapshottable directory.
/// </exception>
public virtual INodeDirectory GetSnapshottableRoot(INodesInPath iip)
{
string path = iip.GetPath();
INodeDirectory dir = INodeDirectory.ValueOf(iip.GetLastINode(), path);
if (!dir.IsSnapshottable())
{
throw new SnapshotException("Directory is not a snapshottable directory: " + path
);
}
return(dir);
}
/// <summary>Load the created list from fsimage.</summary>
/// <param name="parent">The directory that the created list belongs to.</param>
/// <param name="in">
/// The
/// <see cref="System.IO.DataInput"/>
/// to read.
/// </param>
/// <returns>The created list.</returns>
/// <exception cref="System.IO.IOException"/>
private static IList <INode> LoadCreatedList(INodeDirectory parent, DataInput @in)
{
// read the size of the created list
int createdSize = @in.ReadInt();
IList <INode> createdList = new AList <INode>(createdSize);
for (int i = 0; i < createdSize; i++)
{
byte[] createdNodeName = FSImageSerialization.ReadLocalName(@in);
INode created = LoadCreated(createdNodeName, parent);
createdList.AddItem(created);
}
return(createdList);
}
/// <summary>
/// Load the
/// <see cref="Org.Apache.Hadoop.Hdfs.Tools.Snapshot.SnapshotDiff"/>
/// list for the INodeDirectoryWithSnapshot
/// directory.
/// </summary>
/// <param name="dir">The snapshottable directory for loading.</param>
/// <param name="in">
/// The
/// <see cref="System.IO.DataInput"/>
/// instance to read.
/// </param>
/// <param name="loader">The loader</param>
/// <exception cref="System.IO.IOException"/>
public static void LoadDirectoryDiffList(INodeDirectory dir, DataInput @in, FSImageFormat.Loader
loader)
{
int size = @in.ReadInt();
if (dir.IsWithSnapshot())
{
DirectoryWithSnapshotFeature.DirectoryDiffList diffs = dir.GetDiffs();
for (int i = 0; i < size; i++)
{
diffs.AddFirst(LoadDirectoryDiff(dir, @in, loader));
}
}
}
/// <exception cref="System.IO.IOException"/>
private void SerializeDirDiffList(INodeDirectory dir, IList <INodeReference> refList
, OutputStream @out)
{
DirectoryWithSnapshotFeature sf = dir.GetDirectoryWithSnapshotFeature();
if (sf != null)
{
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = sf.GetDiffs().AsList
();
FsImageProto.SnapshotDiffSection.DiffEntry entry = ((FsImageProto.SnapshotDiffSection.DiffEntry
)FsImageProto.SnapshotDiffSection.DiffEntry.NewBuilder().SetInodeId(dir.GetId())
.SetType(FsImageProto.SnapshotDiffSection.DiffEntry.Type.Directorydiff).SetNumOfDiff
(diffList.Count).Build());
entry.WriteDelimitedTo(@out);
for (int i = diffList.Count - 1; i >= 0; i--)
{
// reverse order!
DirectoryWithSnapshotFeature.DirectoryDiff diff = diffList[i];
FsImageProto.SnapshotDiffSection.DirectoryDiff.Builder db = FsImageProto.SnapshotDiffSection.DirectoryDiff
.NewBuilder().SetSnapshotId(diff.GetSnapshotId()).SetChildrenSize(diff.GetChildrenSize
()).SetIsSnapshotRoot(diff.IsSnapshotRoot());
INodeDirectoryAttributes copy = diff.snapshotINode;
if (!diff.IsSnapshotRoot() && copy != null)
{
db.SetName(ByteString.CopyFrom(copy.GetLocalNameBytes())).SetSnapshotCopy(FSImageFormatPBINode.Saver.BuildINodeDirectory
(copy, parent.GetSaverContext()));
}
// process created list and deleted list
IList <INode> created = diff.GetChildrenDiff().GetList(Diff.ListType.Created);
db.SetCreatedListSize(created.Count);
IList <INode> deleted = diff.GetChildrenDiff().GetList(Diff.ListType.Deleted);
foreach (INode d in deleted)
{
if (d.IsReference())
{
refList.AddItem(d.AsReference());
db.AddDeletedINodeRef(refList.Count - 1);
}
else
{
db.AddDeletedINode(d.GetId());
}
}
((FsImageProto.SnapshotDiffSection.DirectoryDiff)db.Build()).WriteDelimitedTo(@out
);
SaveCreatedList(created, @out);
}
}
}
/// <summary>Test that the global limit on snapshots is honored.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotLimits()
{
// Setup mock objects for SnapshotManager.createSnapshot.
//
INodeDirectory ids = Org.Mockito.Mockito.Mock <INodeDirectory>();
FSDirectory fsdir = Org.Mockito.Mockito.Mock <FSDirectory>();
INodesInPath iip = Org.Mockito.Mockito.Mock <INodesInPath>();
SnapshotManager sm = Org.Mockito.Mockito.Spy(new SnapshotManager(fsdir));
Org.Mockito.Mockito.DoReturn(ids).When(sm).GetSnapshottableRoot((INodesInPath)Matchers.AnyObject
());
Org.Mockito.Mockito.DoReturn(testMaxSnapshotLimit).When(sm).GetMaxSnapshotID();
// Create testMaxSnapshotLimit snapshots. These should all succeed.
//
for (int i = 0; i < testMaxSnapshotLimit; ++i)
{
sm.CreateSnapshot(iip, "dummy", i.ToString());
}
// Attempt to create one more snapshot. This should fail due to snapshot
// ID rollover.
//
try
{
sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot");
NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown");
}
catch (SnapshotException se)
{
NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover"
));
}
// Delete a snapshot to free up a slot.
//
sm.DeleteSnapshot(iip, string.Empty, Org.Mockito.Mockito.Mock <INode.BlocksMapUpdateInfo
>(), new AList <INode>());
// Attempt to create a snapshot again. It should still fail due
// to snapshot ID rollover.
//
try
{
sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot2");
NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown");
}
catch (SnapshotException se)
{
NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover"
));
}
}
/// <summary>Load the created list in a DirectoryDiff</summary>
/// <exception cref="System.IO.IOException"/>
private IList <INode> LoadCreatedList(InputStream @in, INodeDirectory dir, int size
)
{
IList <INode> clist = new AList <INode>(size);
for (long c = 0; c < size; c++)
{
FsImageProto.SnapshotDiffSection.CreatedListEntry entry = FsImageProto.SnapshotDiffSection.CreatedListEntry
.ParseDelimitedFrom(@in);
INode created = SnapshotFSImageFormat.LoadCreated(entry.GetName().ToByteArray(),
dir);
clist.AddItem(created);
}
return(clist);
}
/// <summary>Add an inode into parent's children list.</summary>
/// <remarks>
/// Add an inode into parent's children list. The caller of this method needs
/// to make sure that parent is in the given snapshot "latest".
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
public virtual bool AddChild(INodeDirectory parent, INode inode, bool setModTime,
int latestSnapshotId)
{
DirectoryWithSnapshotFeature.ChildrenDiff diff = diffs.CheckAndAddLatestSnapshotDiff
(latestSnapshotId, parent).diff;
int undoInfo = diff.Create(inode);
bool added = parent.AddChild(inode, setModTime, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
if (!added)
{
diff.UndoCreate(inode, undoInfo);
}
return(added);
}
/// <summary>Find the snapshot matching the given name.</summary>
/// <param name="snapshotRoot">The directory where snapshots were taken.</param>
/// <param name="snapshotName">The name of the snapshot.</param>
/// <returns>The corresponding snapshot. Null if snapshotName is null or empty.</returns>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException">
/// If snapshotName is not null or empty, but there
/// is no snapshot matching the name.
/// </exception>
private Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot GetSnapshotByName
(INodeDirectory snapshotRoot, string snapshotName)
{
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s = null;
if (snapshotName != null && !snapshotName.IsEmpty())
{
int index = SearchSnapshot(DFSUtil.String2Bytes(snapshotName));
if (index < 0)
{
throw new SnapshotException("Cannot find the snapshot of directory " + snapshotRoot
.GetFullPathName() + " with name " + snapshotName);
}
s = snapshotsByNames[index];
}
return(s);
}
/// <summary>
/// Test snapshot during file appending, before the corresponding
/// <see cref="Org.Apache.Hadoop.FS.FSDataOutputStream"/>
/// instance closes.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotWhileAppending()
{
Path file = new Path(dir, "file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// 1. append without closing stream --> create snapshot
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot should be stored into s0's
// deleted list, with size BLOCKSIZE*2
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize());
INodeDirectory dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
DirectoryWithSnapshotFeature.DirectoryDiff last = dirNode.GetDiffs().GetLast();
// 2. append without closing stream
@out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
// re-check nodeInDeleted_S0
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 3. take snapshot --> close stream
hdfs.CreateSnapshot(dir, "s1");
@out.Close();
// check: an INodeFileUnderConstructionWithSnapshot with size BLOCKSIZE*3 should
// have been stored in s1's deleted list
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
dirNode = fsdir.GetINode(dir.ToString()).AsDirectory();
last = dirNode.GetDiffs().GetLast();
NUnit.Framework.Assert.IsTrue(fileNode.IsWithSnapshot());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
// 4. modify file --> append without closing stream --> take snapshot -->
// close stream
hdfs.SetReplication(file, (short)(Replication - 1));
@out = AppendFileWithoutClosing(file, Blocksize);
hdfs.CreateSnapshot(dir, "s2");
@out.Close();
// re-check the size of nodeInDeleted_S1
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId
()));
}
/// <summary>
/// Find the latest snapshot that 1) covers the given inode (which means the
/// snapshot was either taken on the inode or taken on an ancestor of the
/// inode), and 2) was taken before the given snapshot (if the given snapshot
/// is not null).
/// </summary>
/// <param name="inode">the given inode that the returned snapshot needs to cover</param>
/// <param name="anchor">the returned snapshot should be taken before this given id.</param>
/// <returns>
/// id of the latest snapshot that covers the given inode and was taken
/// before the the given snapshot (if it is not null).
/// </returns>
public static int FindLatestSnapshot(INode inode, int anchor)
{
int latest = NoSnapshotId;
for (; inode != null; inode = inode.GetParent())
{
if (inode.IsDirectory())
{
INodeDirectory dir = inode.AsDirectory();
if (dir.IsWithSnapshot())
{
latest = dir.GetDiffs().UpdatePrior(anchor, latest);
}
}
}
return(latest);
}
/// <summary>Used to record the modification of a symlink node</summary>
public virtual INode SaveChild2Snapshot(INodeDirectory currentINode, INode child,
int latestSnapshotId, INode snapshotCopy)
{
Preconditions.CheckArgument(!child.IsDirectory(), "child is a directory, child=%s"
, child);
Preconditions.CheckArgument(latestSnapshotId != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId);
DirectoryWithSnapshotFeature.DirectoryDiff diff = diffs.CheckAndAddLatestSnapshotDiff
(latestSnapshotId, currentINode);
if (diff.GetChild(child.GetLocalNameBytes(), false, currentINode) != null)
{
// it was already saved in the latest snapshot earlier.
return(child);
}
diff.diff.Modify(snapshotCopy, child);
return(child);
}
