/// <summary>Load the snapshots section from fsimage.</summary>
/// <remarks>
/// Load the snapshots section from fsimage. Also add snapshottable feature
/// to snapshottable directories.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public void LoadSnapshotSection(InputStream @in)
{
SnapshotManager sm = fsn.GetSnapshotManager();
FsImageProto.SnapshotSection section = FsImageProto.SnapshotSection.ParseDelimitedFrom
(@in);
int snum = section.GetNumSnapshots();
sm.SetNumSnapshots(snum);
sm.SetSnapshotCounter(section.GetSnapshotCounter());
foreach (long sdirId in section.GetSnapshottableDirList())
{
INodeDirectory dir = fsDir.GetInode(sdirId).AsDirectory();
if (!dir.IsSnapshottable())
{
dir.AddSnapshottableFeature();
}
else
{
// dir is root, and admin set root to snapshottable before
dir.SetSnapshotQuota(DirectorySnapshottableFeature.SnapshotLimit);
}
sm.AddSnapshottable(dir);
}
LoadSnapshots(@in, snum);
}
/// <summary>Test that the global limit on snapshots is honored.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotLimits()
{
// Setup mock objects for SnapshotManager.createSnapshot.
//
INodeDirectory ids = Org.Mockito.Mockito.Mock <INodeDirectory>();
FSDirectory fsdir = Org.Mockito.Mockito.Mock <FSDirectory>();
INodesInPath iip = Org.Mockito.Mockito.Mock <INodesInPath>();
SnapshotManager sm = Org.Mockito.Mockito.Spy(new SnapshotManager(fsdir));
Org.Mockito.Mockito.DoReturn(ids).When(sm).GetSnapshottableRoot((INodesInPath)Matchers.AnyObject
());
Org.Mockito.Mockito.DoReturn(testMaxSnapshotLimit).When(sm).GetMaxSnapshotID();
// Create testMaxSnapshotLimit snapshots. These should all succeed.
//
for (int i = 0; i < testMaxSnapshotLimit; ++i)
{
sm.CreateSnapshot(iip, "dummy", i.ToString());
}
// Attempt to create one more snapshot. This should fail due to snapshot
// ID rollover.
//
try
{
sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot");
NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown");
}
catch (SnapshotException se)
{
NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover"
));
}
// Delete a snapshot to free up a slot.
//
sm.DeleteSnapshot(iip, string.Empty, Org.Mockito.Mockito.Mock <INode.BlocksMapUpdateInfo
>(), new AList <INode>());
// Attempt to create a snapshot again. It should still fail due
// to snapshot ID rollover.
//
try
{
sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot2");
NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown");
}
catch (SnapshotException se)
{
NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover"
));
}
}
/// <summary>save all the snapshottable directories and snapshots to fsimage</summary>
/// <exception cref="System.IO.IOException"/>
public void SerializeSnapshotSection(OutputStream @out)
{
SnapshotManager sm = fsn.GetSnapshotManager();
FsImageProto.SnapshotSection.Builder b = FsImageProto.SnapshotSection.NewBuilder(
).SetSnapshotCounter(sm.GetSnapshotCounter()).SetNumSnapshots(sm.GetNumSnapshots
());
INodeDirectory[] snapshottables = sm.GetSnapshottableDirs();
foreach (INodeDirectory sdir in snapshottables)
{
b.AddSnapshottableDir(sdir.GetId());
}
((FsImageProto.SnapshotSection)b.Build()).WriteDelimitedTo(@out);
int i = 0;
foreach (INodeDirectory sdir_1 in snapshottables)
{
foreach (Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s in sdir_1.GetDirectorySnapshottableFeature
().GetSnapshotList())
{
Snapshot.Root sroot = s.GetRoot();
FsImageProto.SnapshotSection.Snapshot.Builder sb = FsImageProto.SnapshotSection.Snapshot
.NewBuilder().SetSnapshotId(s.GetId());
FsImageProto.INodeSection.INodeDirectory.Builder db = FSImageFormatPBINode.Saver.BuildINodeDirectory
(sroot, parent.GetSaverContext());
FsImageProto.INodeSection.INode r = ((FsImageProto.INodeSection.INode)FsImageProto.INodeSection.INode
.NewBuilder().SetId(sroot.GetId()).SetType(FsImageProto.INodeSection.INode.Type.
Directory).SetName(ByteString.CopyFrom(sroot.GetLocalNameBytes())).SetDirectory(
db).Build());
((FsImageProto.SnapshotSection.Snapshot)sb.SetRoot(r).Build()).WriteDelimitedTo(@out
);
i++;
if (i % FSImageFormatProtobuf.Saver.CheckCancelInterval == 0)
{
context.CheckCancelled();
}
}
}
Preconditions.CheckState(i == sm.GetNumSnapshots());
parent.CommitSection(headers, FSImageFormatProtobuf.SectionName.Snapshot);
}
public virtual void TestSnapshotStatsMXBeanInfo()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
string pathName = "/snapshot";
Path path = new Path(pathName);
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
SnapshotManager sm = cluster.GetNamesystem().GetSnapshotManager();
DistributedFileSystem dfs = (DistributedFileSystem)cluster.GetFileSystem();
dfs.Mkdirs(path);
dfs.AllowSnapshot(path);
dfs.CreateSnapshot(path);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo"
);
CompositeData[] directories = (CompositeData[])mbs.GetAttribute(mxbeanName, "SnapshottableDirectories"
);
int numDirectories = Sharpen.Runtime.GetArrayLength(directories);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshottableDirs(), numDirectories);
CompositeData[] snapshots = (CompositeData[])mbs.GetAttribute(mxbeanName, "Snapshots"
);
int numSnapshots = Sharpen.Runtime.GetArrayLength(snapshots);
NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshots(), numSnapshots);
CompositeData d = (CompositeData)Sharpen.Runtime.GetArrayValue(directories, 0);
CompositeData s = (CompositeData)Sharpen.Runtime.GetArrayValue(snapshots, 0);
NUnit.Framework.Assert.IsTrue(((string)d.Get("path")).Contains(pathName));
NUnit.Framework.Assert.IsTrue(((string)s.Get("snapshotDirectory")).Contains(pathName
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestCheckpoint()
{
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
secondary = new SecondaryNameNode(conf);
SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager();
SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager
();
FileSystem fs = cluster.GetFileSystem();
HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 1. Create a snapshottable directory foo on the NN.
fs.Mkdirs(TestPath);
admin.AllowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 2. Create a snapshot of the dir foo. This will be referenced both in
// the SnapshotManager as well as in the file system tree. The snapshot
// count will go up to 1.
Path snapshotPath = fs.CreateSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs());
// 3. Start up a 2NN and have it do a checkpoint. It will have foo and its
// snapshot in its list of snapshottable dirs referenced from the
// SnapshotManager, as well as in the file system tree.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs
());
// 4. Disallow snapshots on and delete foo on the NN. The snapshot count
// will go down to 0 and the snapshottable dir will be removed from the fs
// tree.
fs.DeleteSnapshot(TestPath, snapshotPath.GetName());
admin.DisallowSnapshot(TestPath);
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs());
// 5. Have the NN do a saveNamespace, writing out a new fsimage with
// snapshot count 0.
NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false);
NameNodeAdapter.SaveNamespace(cluster.GetNameNode());
NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode());
// 6. Have the still-running 2NN do a checkpoint. It will notice that the
// fsimage has changed on the NN and redownload/reload from that image.
// This will replace all INodes in the file system tree as well as reset
// the snapshot counter to 0 in the SnapshotManager. However, it will not
// clear the list of snapshottable dirs referenced from the
// SnapshotManager. When it writes out an fsimage, the 2NN will write out
// 0 for the snapshot count, but still serialize the snapshottable dir
// referenced in the SnapshotManager even though it no longer appears in
// the file system tree. The NN will not be able to start up with this.
secondary.DoCheckpoint();
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots());
NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs
());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (secondary != null)
{
secondary.Shutdown();
}
}
}
