/// <summary>Load the snapshots section from fsimage.</summary> /// <remarks> /// Load the snapshots section from fsimage. Also add snapshottable feature /// to snapshottable directories. /// </remarks> /// <exception cref="System.IO.IOException"/> public void LoadSnapshotSection(InputStream @in) { SnapshotManager sm = fsn.GetSnapshotManager(); FsImageProto.SnapshotSection section = FsImageProto.SnapshotSection.ParseDelimitedFrom (@in); int snum = section.GetNumSnapshots(); sm.SetNumSnapshots(snum); sm.SetSnapshotCounter(section.GetSnapshotCounter()); foreach (long sdirId in section.GetSnapshottableDirList()) { INodeDirectory dir = fsDir.GetInode(sdirId).AsDirectory(); if (!dir.IsSnapshottable()) { dir.AddSnapshottableFeature(); } else { // dir is root, and admin set root to snapshottable before dir.SetSnapshotQuota(DirectorySnapshottableFeature.SnapshotLimit); } sm.AddSnapshottable(dir); } LoadSnapshots(@in, snum); }
/// <summary>Test that the global limit on snapshots is honored.</summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotLimits() { // Setup mock objects for SnapshotManager.createSnapshot. // INodeDirectory ids = Org.Mockito.Mockito.Mock <INodeDirectory>(); FSDirectory fsdir = Org.Mockito.Mockito.Mock <FSDirectory>(); INodesInPath iip = Org.Mockito.Mockito.Mock <INodesInPath>(); SnapshotManager sm = Org.Mockito.Mockito.Spy(new SnapshotManager(fsdir)); Org.Mockito.Mockito.DoReturn(ids).When(sm).GetSnapshottableRoot((INodesInPath)Matchers.AnyObject ()); Org.Mockito.Mockito.DoReturn(testMaxSnapshotLimit).When(sm).GetMaxSnapshotID(); // Create testMaxSnapshotLimit snapshots. These should all succeed. // for (int i = 0; i < testMaxSnapshotLimit; ++i) { sm.CreateSnapshot(iip, "dummy", i.ToString()); } // Attempt to create one more snapshot. This should fail due to snapshot // ID rollover. // try { sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot"); NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown"); } catch (SnapshotException se) { NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover" )); } // Delete a snapshot to free up a slot. // sm.DeleteSnapshot(iip, string.Empty, Org.Mockito.Mockito.Mock <INode.BlocksMapUpdateInfo >(), new AList <INode>()); // Attempt to create a snapshot again. It should still fail due // to snapshot ID rollover. // try { sm.CreateSnapshot(iip, "dummy", "shouldFailSnapshot2"); NUnit.Framework.Assert.Fail("Expected SnapshotException not thrown"); } catch (SnapshotException se) { NUnit.Framework.Assert.IsTrue(StringUtils.ToLowerCase(se.Message).Contains("rollover" )); } }
/// <summary>save all the snapshottable directories and snapshots to fsimage</summary> /// <exception cref="System.IO.IOException"/> public void SerializeSnapshotSection(OutputStream @out) { SnapshotManager sm = fsn.GetSnapshotManager(); FsImageProto.SnapshotSection.Builder b = FsImageProto.SnapshotSection.NewBuilder( ).SetSnapshotCounter(sm.GetSnapshotCounter()).SetNumSnapshots(sm.GetNumSnapshots ()); INodeDirectory[] snapshottables = sm.GetSnapshottableDirs(); foreach (INodeDirectory sdir in snapshottables) { b.AddSnapshottableDir(sdir.GetId()); } ((FsImageProto.SnapshotSection)b.Build()).WriteDelimitedTo(@out); int i = 0; foreach (INodeDirectory sdir_1 in snapshottables) { foreach (Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s in sdir_1.GetDirectorySnapshottableFeature ().GetSnapshotList()) { Snapshot.Root sroot = s.GetRoot(); FsImageProto.SnapshotSection.Snapshot.Builder sb = FsImageProto.SnapshotSection.Snapshot .NewBuilder().SetSnapshotId(s.GetId()); FsImageProto.INodeSection.INodeDirectory.Builder db = FSImageFormatPBINode.Saver.BuildINodeDirectory (sroot, parent.GetSaverContext()); FsImageProto.INodeSection.INode r = ((FsImageProto.INodeSection.INode)FsImageProto.INodeSection.INode .NewBuilder().SetId(sroot.GetId()).SetType(FsImageProto.INodeSection.INode.Type. Directory).SetName(ByteString.CopyFrom(sroot.GetLocalNameBytes())).SetDirectory( db).Build()); ((FsImageProto.SnapshotSection.Snapshot)sb.SetRoot(r).Build()).WriteDelimitedTo(@out ); i++; if (i % FSImageFormatProtobuf.Saver.CheckCancelInterval == 0) { context.CheckCancelled(); } } } Preconditions.CheckState(i == sm.GetNumSnapshots()); parent.CommitSection(headers, FSImageFormatProtobuf.SectionName.Snapshot); }
public virtual void TestSnapshotStatsMXBeanInfo() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; string pathName = "/snapshot"; Path path = new Path(pathName); try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); SnapshotManager sm = cluster.GetNamesystem().GetSnapshotManager(); DistributedFileSystem dfs = (DistributedFileSystem)cluster.GetFileSystem(); dfs.Mkdirs(path); dfs.AllowSnapshot(path); dfs.CreateSnapshot(path); MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer(); ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo" ); CompositeData[] directories = (CompositeData[])mbs.GetAttribute(mxbeanName, "SnapshottableDirectories" ); int numDirectories = Sharpen.Runtime.GetArrayLength(directories); NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshottableDirs(), numDirectories); CompositeData[] snapshots = (CompositeData[])mbs.GetAttribute(mxbeanName, "Snapshots" ); int numSnapshots = Sharpen.Runtime.GetArrayLength(snapshots); NUnit.Framework.Assert.AreEqual(sm.GetNumSnapshots(), numSnapshots); CompositeData d = (CompositeData)Sharpen.Runtime.GetArrayValue(directories, 0); CompositeData s = (CompositeData)Sharpen.Runtime.GetArrayValue(snapshots, 0); NUnit.Framework.Assert.IsTrue(((string)d.Get("path")).Contains(pathName)); NUnit.Framework.Assert.IsTrue(((string)s.Get("snapshotDirectory")).Contains(pathName )); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestCheckpoint() { MiniDFSCluster cluster = null; SecondaryNameNode secondary = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); secondary = new SecondaryNameNode(conf); SnapshotManager nnSnapshotManager = cluster.GetNamesystem().GetSnapshotManager(); SnapshotManager secondarySnapshotManager = secondary.GetFSNamesystem().GetSnapshotManager (); FileSystem fs = cluster.GetFileSystem(); HdfsAdmin admin = new HdfsAdmin(FileSystem.GetDefaultUri(conf), conf); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 1. Create a snapshottable directory foo on the NN. fs.Mkdirs(TestPath); admin.AllowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 2. Create a snapshot of the dir foo. This will be referenced both in // the SnapshotManager as well as in the file system tree. The snapshot // count will go up to 1. Path snapshotPath = fs.CreateSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, nnSnapshotManager.GetNumSnapshottableDirs()); // 3. Start up a 2NN and have it do a checkpoint. It will have foo and its // snapshot in its list of snapshottable dirs referenced from the // SnapshotManager, as well as in the file system tree. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(1, secondarySnapshotManager.GetNumSnapshottableDirs ()); // 4. Disallow snapshots on and delete foo on the NN. The snapshot count // will go down to 0 and the snapshottable dir will be removed from the fs // tree. fs.DeleteSnapshot(TestPath, snapshotPath.GetName()); admin.DisallowSnapshot(TestPath); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, nnSnapshotManager.GetNumSnapshottableDirs()); // 5. Have the NN do a saveNamespace, writing out a new fsimage with // snapshot count 0. NameNodeAdapter.EnterSafeMode(cluster.GetNameNode(), false); NameNodeAdapter.SaveNamespace(cluster.GetNameNode()); NameNodeAdapter.LeaveSafeMode(cluster.GetNameNode()); // 6. Have the still-running 2NN do a checkpoint. It will notice that the // fsimage has changed on the NN and redownload/reload from that image. // This will replace all INodes in the file system tree as well as reset // the snapshot counter to 0 in the SnapshotManager. However, it will not // clear the list of snapshottable dirs referenced from the // SnapshotManager. When it writes out an fsimage, the 2NN will write out // 0 for the snapshot count, but still serialize the snapshottable dir // referenced in the SnapshotManager even though it no longer appears in // the file system tree. The NN will not be able to start up with this. secondary.DoCheckpoint(); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshots()); NUnit.Framework.Assert.AreEqual(0, secondarySnapshotManager.GetNumSnapshottableDirs ()); } finally { if (cluster != null) { cluster.Shutdown(); } if (secondary != null) { secondary.Shutdown(); } } }