public virtual void TestNoLogs()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     RunTest(tc);
 }
 public virtual void TestPurgeLessThanRetention()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), false);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
 public virtual void TestPurgeEasyCase()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     // Test that other files don't get purged
     tc.AddLog("/foo1/current/VERSION", false);
     RunTest(tc);
 }
Exemplo n.º 4
0
 public static void AssertNNHasCheckpoints(MiniDFSCluster cluster, int nnIdx, IList
                                           <int> txids)
 {
     foreach (FilePath nameDir in GetNameNodeCurrentDirs(cluster, nnIdx))
     {
         Log.Info("examining name dir with files: " + Joiner.On(",").Join(nameDir.ListFiles
                                                                              ()));
         // Should have fsimage_N for the three checkpoints
         Log.Info("Examining storage dir " + nameDir + " with contents: " + StringUtils.Join
                      (nameDir.ListFiles(), ", "));
         foreach (long checkpointTxId in txids)
         {
             FilePath image = new FilePath(nameDir, NNStorage.GetImageFileName(checkpointTxId)
                                           );
             NUnit.Framework.Assert.IsTrue("Expected non-empty " + image, image.Length() > 0);
         }
     }
 }
Exemplo n.º 5
0
        public virtual void TestCurrentStorageInspector()
        {
            FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
                                                                 ();

            Storage.StorageDirectory mockDir = FSImageTestUtil.MockStorageDirectory(NNStorage.NameNodeDirType
                                                                                    .ImageAndEdits, false, "/foo/current/" + NNStorage.GetImageFileName(123), "/foo/current/"
                                                                                    + NNStorage.GetFinalizedEditsFileName(123, 456), "/foo/current/" + NNStorage.GetImageFileName
                                                                                        (456), "/foo/current/" + NNStorage.GetInProgressEditsFileName(457));
            inspector.InspectDirectory(mockDir);
            NUnit.Framework.Assert.AreEqual(2, inspector.foundImages.Count);
            FSImageStorageInspector.FSImageFile latestImage = inspector.GetLatestImages()[0];
            NUnit.Framework.Assert.AreEqual(456, latestImage.txId);
            NUnit.Framework.Assert.AreSame(mockDir, latestImage.sd);
            NUnit.Framework.Assert.IsTrue(inspector.IsUpgradeFinalized());
            NUnit.Framework.Assert.AreEqual(new FilePath("/foo/current/" + NNStorage.GetImageFileName
                                                             (456)), latestImage.GetFile());
        }
 public virtual void TestRetainExtraLogs()
 {
     conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50);
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
     tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
     // Since we need 50 extra edits, *do* retain the 201-300 segment
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
Exemplo n.º 7
0
 /// <summary>For namenode, Verify that the current and previous directories exist.</summary>
 /// <remarks>
 /// For namenode, Verify that the current and previous directories exist.
 /// Verify that previous hasn't been modified by comparing the checksum of all
 /// its files with their original checksum. It is assumed that the
 /// server has recovered and upgraded.
 /// </remarks>
 /// <exception cref="System.IO.IOException"/>
 internal virtual void CheckNameNode(string[] baseDirs, long imageTxId)
 {
     foreach (string baseDir in baseDirs)
     {
         Log.Info("Checking namenode directory " + baseDir);
         Log.Info("==== Contents ====:\n  " + Joiner.On("  \n").Join(new FilePath(baseDir,
                                                                                  "current").List()));
         Log.Info("==================");
         GenericTestUtils.AssertExists(new FilePath(baseDir, "current"));
         GenericTestUtils.AssertExists(new FilePath(baseDir, "current/VERSION"));
         GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetInProgressEditsFileName
                                                        (imageTxId + 1)));
         GenericTestUtils.AssertExists(new FilePath(baseDir, "current/" + NNStorage.GetImageFileName
                                                        (imageTxId)));
         GenericTestUtils.AssertExists(new FilePath(baseDir, "current/seen_txid"));
         FilePath previous = new FilePath(baseDir, "previous");
         GenericTestUtils.AssertExists(previous);
         NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType
                                                                           .NameNode, previous, false), UpgradeUtilities.ChecksumMasterNameNodeContents());
     }
 }
 public virtual void TestRetainExtraLogsLimitedSegments()
 {
     conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150);
     conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2);
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
     tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     // Segments containing txns upto txId 250 are extra and should be purged.
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty"
               , true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt"
               , true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true);
     // Only retain 2 extra segments. The 301-350 and 351-400 segments are
     // considered required, not extra.
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty"
               , false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt"
               , false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
        private void CheckNNStorage(NNStorage storage, long imageTxId, long trashEndTxId)
        {
            IList <FilePath> finalizedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits
                                                               , NNStorage.GetFinalizedEditsFileName(1, imageTxId));

            NUnit.Framework.Assert.IsTrue(FileExists(finalizedEdits));
            IList <FilePath> inprogressEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits
                                                                , NNStorage.GetInProgressEditsFileName(imageTxId + 1));

            // For rollback case we will have an inprogress file for future transactions
            NUnit.Framework.Assert.IsTrue(FileExists(inprogressEdits));
            if (trashEndTxId > 0)
            {
                IList <FilePath> trashedEdits = storage.GetFiles(NNStorage.NameNodeDirType.Edits,
                                                                 NNStorage.GetFinalizedEditsFileName(imageTxId + 1, trashEndTxId) + ".trash");
                NUnit.Framework.Assert.IsTrue(FileExists(trashedEdits));
            }
            string imageFileName = trashEndTxId > 0 ? NNStorage.GetImageFileName(imageTxId) :
                                   NNStorage.GetRollbackImageFileName(imageTxId);
            IList <FilePath> imageFiles = storage.GetFiles(NNStorage.NameNodeDirType.Image, imageFileName
                                                           );

            NUnit.Framework.Assert.IsTrue(FileExists(imageFiles));
        }
Exemplo n.º 10
0
        public virtual void TestStorageRestore()
        {
            int numDatanodes = 0;

            cluster = new MiniDFSCluster.Builder(config).NumDataNodes(numDatanodes).ManageNameDfsDirs
                          (false).Build();
            cluster.WaitActive();
            SecondaryNameNode secondary = new SecondaryNameNode(config);

            System.Console.Out.WriteLine("****testStorageRestore: Cluster and SNN started");
            PrintStorages(cluster.GetNameNode().GetFSImage());
            FileSystem fs   = cluster.GetFileSystem();
            Path       path = new Path("/", "test");

            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
            System.Console.Out.WriteLine("****testStorageRestore: dir 'test' created, invalidating storage..."
                                         );
            InvalidateStorage(cluster.GetNameNode().GetFSImage(), ImmutableSet.Of(path2, path3
                                                                                  ));
            PrintStorages(cluster.GetNameNode().GetFSImage());
            System.Console.Out.WriteLine("****testStorageRestore: storage invalidated");
            path = new Path("/", "test1");
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
            System.Console.Out.WriteLine("****testStorageRestore: dir 'test1' created");
            // We did another edit, so the still-active directory at 'path1'
            // should now differ from the others
            FSImageTestUtil.AssertFileContentsDifferent(2, new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
                                                                            (1)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(1)),
                                                        new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName
                                                                    (1)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
            System.Console.Out.WriteLine("****testStorageRestore: checkfiles(false) run");
            secondary.DoCheckpoint();
            ///should enable storage..
            // We should have a checkpoint through txid 4 in the two image dirs
            // (txid=4 for BEGIN, mkdir, mkdir, END)
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
                                                                    (4)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(4)));
            NUnit.Framework.Assert.IsFalse("Should not have any image in an edits-only directory"
                                           , new FilePath(path3, "current/" + NNStorage.GetImageFileName(4)).Exists());
            // Should have finalized logs in the directory that didn't fail
            NUnit.Framework.Assert.IsTrue("Should have finalized logs in the directory that didn't fail"
                                          , new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
                                              ());
            // Should not have finalized logs in the failed directories
            NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
                                           , new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
                                               ());
            NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
                                           , new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
                                               ());
            // The new log segment should be in all of the directories.
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
                                                                    (5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
                                                   new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
            string md5BeforeEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/"
                                                                           + NNStorage.GetInProgressEditsFileName(5)));

            // The original image should still be the previously failed image
            // directory after it got restored, since it's still useful for
            // a recovery!
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
                                                                    (0)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(0)));
            // Do another edit to verify that all the logs are active.
            path = new Path("/", "test2");
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
            // Logs should be changed by the edit.
            string md5AfterEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/" +
                                                                          NNStorage.GetInProgressEditsFileName(5)));

            NUnit.Framework.Assert.IsFalse(md5BeforeEdit.Equals(md5AfterEdit));
            // And all logs should be changed.
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
                                                                    (5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
                                                   new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
            secondary.Shutdown();
            cluster.Shutdown();
            // All logs should be finalized by clean shutdown
            FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName
                                                                    (5, 7)), new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(5,
                                                                                                                                                  7)), new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(5, 7))
                                                   );
        }
Exemplo n.º 11
0
        /// <summary>Corrupts the MD5 sum of the fsimage.</summary>
        /// <param name="corruptAll">
        /// whether to corrupt one or all of the MD5 sums in the configured
        /// namedirs
        /// </param>
        /// <exception cref="System.IO.IOException"/>
        private void CorruptFSImageMD5(bool corruptAll)
        {
            IList <URI> nameDirs = (IList <URI>)FSNamesystem.GetNamespaceDirs(config);

            // Corrupt the md5 files in all the namedirs
            foreach (URI uri in nameDirs)
            {
                // Directory layout looks like:
                // test/data/dfs/nameN/current/{fsimage,edits,...}
                FilePath nameDir = new FilePath(uri.GetPath());
                FilePath dfsDir  = nameDir.GetParentFile();
                NUnit.Framework.Assert.AreEqual(dfsDir.GetName(), "dfs");
                // make sure we got right dir
                // Set the md5 file to all zeros
                FilePath imageFile = new FilePath(nameDir, Storage.StorageDirCurrent + "/" + NNStorage
                                                  .GetImageFileName(0));
                MD5FileUtils.SaveMD5File(imageFile, new MD5Hash(new byte[16]));
                // Only need to corrupt one if !corruptAll
                if (!corruptAll)
                {
                    break;
                }
            }
        }
        public virtual void TestPurgingWithNameEditsDirAfterFailure()
        {
            MiniDFSCluster cluster = null;
            Configuration  conf    = new HdfsConfiguration();

            conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 0);
            FilePath sd0 = new FilePath(TestRootDir, "nn0");
            FilePath sd1 = new FilePath(TestRootDir, "nn1");
            FilePath cd0 = new FilePath(sd0, "current");
            FilePath cd1 = new FilePath(sd1, "current");

            conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(sd0, sd1));
            try
            {
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
                                                                                             ).Format(true).Build();
                NameNode nn = cluster.GetNameNode();
                DoSaveNamespace(nn);
                Log.Info("After first save, images 0 and 2 should exist in both dirs");
                GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (0), NNStorage.GetImageFileName(2));
                GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (0), NNStorage.GetImageFileName(2));
                GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (1, 2), NNStorage.GetInProgressEditsFileName(3));
                GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (1, 2), NNStorage.GetInProgressEditsFileName(3));
                DoSaveNamespace(nn);
                Log.Info("After second save, image 0 should be purged, " + "and image 4 should exist in both."
                         );
                GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (2), NNStorage.GetImageFileName(4));
                GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (2), NNStorage.GetImageFileName(4));
                GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (3, 4), NNStorage.GetInProgressEditsFileName(5));
                GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (3, 4), NNStorage.GetInProgressEditsFileName(5));
                Log.Info("Failing first storage dir by chmodding it");
                NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "000"));
                DoSaveNamespace(nn);
                Log.Info("Restoring accessibility of first storage dir");
                NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "755"));
                Log.Info("nothing should have been purged in first storage dir");
                GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (2), NNStorage.GetImageFileName(4));
                GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (3, 4), NNStorage.GetInProgressEditsFileName(5));
                Log.Info("fsimage_2 should be purged in second storage dir");
                GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (4), NNStorage.GetImageFileName(6));
                GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (5, 6), NNStorage.GetInProgressEditsFileName(7));
                Log.Info("On next save, we should purge logs from the failed dir," + " but not images, since the image directory is in failed state."
                         );
                DoSaveNamespace(nn);
                GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (6), NNStorage.GetImageFileName(8));
                GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
                                                      (7, 8), NNStorage.GetInProgressEditsFileName(9));
                GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
                                                      (2), NNStorage.GetImageFileName(4));
                GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetInProgressEditsFileName
                                                      (9));
            }
            finally
            {
                FileUtil.Chmod(cd0.GetAbsolutePath(), "755");
                Log.Info("Shutting down...");
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
Exemplo n.º 13
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCancelSaveNamespace()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   image   = fsn.GetFSImage();
            NNStorage storage = image.GetStorage();

            storage.Close();
            // unlock any directories that FSNamesystem's initialization may have locked
            storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs
                                              (conf));
            FSNamesystem spyFsn   = Org.Mockito.Mockito.Spy(fsn);
            FSNamesystem finalFsn = spyFsn;

            GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
            BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager());

            Whitebox.SetInternalState(finalFsn, "blockIdManager", bid);
            Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2();
            ExecutorService pool = Executors.NewFixedThreadPool(2);

            try
            {
                DoAnEdit(fsn, 1);
                Canceler canceler = new Canceler();
                // Save namespace
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler
                                                                              ));
                    // Wait until saveNamespace calls getGenerationStamp
                    delayer.WaitForCall();
                    // then cancel the saveNamespace
                    Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler));
                    // give the cancel call time to run
                    Sharpen.Thread.Sleep(500);
                    // allow saveNamespace to proceed - it should check the cancel flag after
                    // this point and throw an exception
                    delayer.Proceed();
                    cancelFuture.Get();
                    saverFuture.Get();
                    NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!");
                }
                catch (Exception t)
                {
                    GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t);
                }
                Log.Info("Successfully cancelled a saveNamespace");
                // Check that we have only the original image and not any
                // cruft left over from half-finished images
                FSImageTestUtil.LogStorageContents(Log, storage);
                foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
                {
                    FilePath curDir = sd.GetCurrentDir();
                    GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName
                                                          (0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix);
                }
            }
            finally
            {
                fsn.Close();
            }
        }