/// <exception cref="System.IO.IOException"/>
        private int DownloadImage(NNStorage storage, NamenodeProtocol proxy)
        {
            // Load the newly formatted image, using all of the directories
            // (including shared edits)
            long    imageTxId = proxy.GetMostRecentCheckpointTxId();
            long    curTxId   = proxy.GetTransactionID();
            FSImage image     = new FSImage(conf);

            try
            {
                image.GetStorage().SetStorageInfo(storage);
                image.InitEditLog(HdfsServerConstants.StartupOption.Regular);
                System.Diagnostics.Debug.Assert(image.GetEditLog().IsOpenForRead(), "Expected edit log to be open for read"
                                                );
                // Ensure that we have enough edits already in the shared directory to
                // start up from the last checkpoint on the active.
                if (!skipSharedEditsCheck && !CheckLogsAvailableForRead(image, imageTxId, curTxId
                                                                        ))
                {
                    return(ErrCodeLogsUnavailable);
                }
                image.GetStorage().WriteTransactionIdFileToStorage(curTxId);
                // Download that checkpoint into our storage directories.
                MD5Hash hash = TransferFsImage.DownloadImageToStorage(otherHttpAddr, imageTxId, storage
                                                                      , true);
                image.SaveDigestAndRenameCheckpointImage(NNStorage.NameNodeFile.Image, imageTxId,
                                                         hash);
            }
            catch (IOException ioe)
            {
                image.Close();
                throw;
            }
            return(0);
        }
        /// <summary>Injects a failure on all storage directories while saving namespace.</summary>
        /// <param name="restoreStorageAfterFailure">
        /// if true, will try to save again after
        /// clearing the failure injection
        /// </param>
        /// <exception cref="System.Exception"/>
        public virtual void DoTestFailedSaveNamespace(bool restoreStorageAfterFailure)
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();

            storage.Close();
            // unlock any directories that FSNamesystem's initialization may have locked
            NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);

            originalImage.storage = spyStorage;
            FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            spyImage.storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem
                                                   .GetNamespaceEditsDirs(conf));
            Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: saveFSImage")).When(
                spyImage).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                   )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
            try
            {
                DoAnEdit(fsn, 1);
                // Save namespace
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    fsn.SaveNamespace();
                    NUnit.Framework.Assert.Fail("saveNamespace did not fail even when all directories failed!"
                                                );
                }
                catch (IOException ioe)
                {
                    Log.Info("Got expected exception", ioe);
                }
                // Ensure that, if storage dirs come back online, things work again.
                if (restoreStorageAfterFailure)
                {
                    Org.Mockito.Mockito.Reset(spyImage);
                    spyStorage.SetRestoreFailedStorage(true);
                    fsn.SaveNamespace();
                    CheckEditExists(fsn, 1);
                }
                // Now shut down and restart the NN
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edits.
                CheckEditExists(fsn, 1);
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
        /// <exception cref="System.Exception"/>
        private void SaveNamespaceWithInjectedFault(TestSaveNamespace.Fault fault)
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();
            NNStorage spyStorage    = Org.Mockito.Mockito.Spy(storage);

            originalImage.storage = spyStorage;
            FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            bool shouldFail = false;

            switch (fault)
            {
            case TestSaveNamespace.Fault.SaveSecondFsimageRte:
            {
                // should we expect the save operation to fail
                // inject fault
                // The spy throws a RuntimeException when writing to the second directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyImage
                                                                                               ).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                                                                                          )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = false;
                break;
            }

            case TestSaveNamespace.Fault.SaveSecondFsimageIoe:
            {
                // The spy throws an IOException when writing to the second directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(false)).When(spyImage
                                                                                                ).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                                                                                           )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = false;
                break;
            }

            case TestSaveNamespace.Fault.SaveAllFsimages:
            {
                // The spy throws IOException in all directories
                Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyImage).SaveFSImage
                    ((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory)Matchers.AnyObject
                        (), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = true;
                break;
            }

            case TestSaveNamespace.Fault.WriteStorageAll:
            {
                // The spy throws an exception before writing any VERSION files
                Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyStorage).WriteAll
                    ();
                shouldFail = true;
                break;
            }

            case TestSaveNamespace.Fault.WriteStorageOne:
            {
                // The spy throws on exception on one particular storage directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyStorage
                                                                                               ).WriteProperties((Storage.StorageDirectory)Matchers.AnyObject());
                // TODO: unfortunately this fails -- should be improved.
                // See HDFS-2173.
                shouldFail = true;
                break;
            }
            }
            try
            {
                DoAnEdit(fsn, 1);
                // Save namespace - this may fail, depending on fault injected
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    fsn.SaveNamespace();
                    if (shouldFail)
                    {
                        NUnit.Framework.Assert.Fail("Did not fail!");
                    }
                }
                catch (Exception e)
                {
                    if (!shouldFail)
                    {
                        throw;
                    }
                    else
                    {
                        Log.Info("Test caught expected exception", e);
                    }
                }
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
                // Should still be able to perform edits
                DoAnEdit(fsn, 2);
                // Now shut down and restart the namesystem
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edits.
                CheckEditExists(fsn, 1);
                CheckEditExists(fsn, 2);
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
        /// <summary>
        /// Verify that a saveNamespace command brings faulty directories
        /// in fs.name.dir and fs.edit.dir back online.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestReinsertnamedirsInSavenamespace()
        {
            // create a configuration with the key to restore error
            // directories in fs.name.dir
            Configuration conf = GetConf();

            conf.SetBoolean(DFSConfigKeys.DfsNamenodeNameDirRestoreKey, true);
            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();
            FSImage   spyImage      = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            FileSystem   fs             = FileSystem.GetLocal(conf);
            FilePath     rootDir        = storage.GetStorageDir(0).GetRoot();
            Path         rootPath       = new Path(rootDir.GetPath(), "current");
            FsPermission permissionNone = new FsPermission((short)0);
            FsPermission permissionAll  = new FsPermission(FsAction.All, FsAction.ReadExecute,
                                                           FsAction.ReadExecute);

            fs.SetPermission(rootPath, permissionNone);
            try
            {
                DoAnEdit(fsn, 1);
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                // Save namespace - should mark the first storage dir as faulty
                // since it's not traversable.
                Log.Info("Doing the first savenamespace.");
                fsn.SaveNamespace();
                Log.Info("First savenamespace sucessful.");
                NUnit.Framework.Assert.IsTrue("Savenamespace should have marked one directory as bad."
                                              + " But found " + storage.GetRemovedStorageDirs().Count + " bad directories.",
                                              storage.GetRemovedStorageDirs().Count == 1);
                fs.SetPermission(rootPath, permissionAll);
                // The next call to savenamespace should try inserting the
                // erroneous directory back to fs.name.dir. This command should
                // be successful.
                Log.Info("Doing the second savenamespace.");
                fsn.SaveNamespace();
                Log.Warn("Second savenamespace sucessful.");
                NUnit.Framework.Assert.IsTrue("Savenamespace should have been successful in removing "
                                              + " bad directories from Image." + " But found " + storage.GetRemovedStorageDirs
                                                  ().Count + " bad directories.", storage.GetRemovedStorageDirs().Count == 0);
                // Now shut down and restart the namesystem
                Log.Info("Shutting down fsimage.");
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                Log.Info("Loading new FSmage from disk.");
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edit.
                Log.Info("Checking reloaded image.");
                CheckEditExists(fsn, 1);
                Log.Info("Reloaded image is good.");
            }
            finally
            {
                if (rootDir.Exists())
                {
                    fs.SetPermission(rootPath, permissionAll);
                }
                if (fsn != null)
                {
                    try
                    {
                        fsn.Close();
                    }
                    catch (Exception t)
                    {
                        Log.Fatal("Failed to shut down", t);
                    }
                }
            }
        }