Exemplo n.º 1
0
        public virtual void TestSaveImageWhileSyncInProgress()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);

            try
            {
                FSImage   fsimage = namesystem.GetFSImage();
                FSEditLog editLog = fsimage.GetEditLog();
                JournalSet.JournalAndStream jas     = editLog.GetJournals()[0];
                EditLogFileOutputStream     spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream
                                                                               )jas.GetCurrentStream());
                jas.SetCurrentStreamForTests(spyElos);
                AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
                CountDownLatch waitToEnterFlush = new CountDownLatch(1);
                Sharpen.Thread doAnEditThread   = new _Thread_371(namesystem, deferredException, waitToEnterFlush
                                                                  );
                Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush);
                // Signal to main thread that the edit thread is in the racy section
                Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush();
                doAnEditThread.Start();
                // Wait for the edit thread to get to the logsync unsynchronized section
                Log.Info("Main thread: waiting to enter flush...");
                waitToEnterFlush.Await();
                NUnit.Framework.Assert.IsNull(deferredException.Get());
                Log.Info("Main thread: detected that logSync is in unsynchronized section.");
                Log.Info("Trying to enter safe mode.");
                Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long"
                         );
                long st = Time.Now();
                namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                long et = Time.Now();
                Log.Info("Entered safe mode");
                // Make sure we really waited for the flush to complete!
                NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
                // Once we're in safe mode, save namespace.
                namesystem.SaveNamespace();
                Log.Info("Joining on edit thread...");
                doAnEditThread.Join();
                NUnit.Framework.Assert.IsNull(deferredException.Get());
                // We did 3 edits: begin, txn, and end
                NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
                                                                  GetFinalizedEditsFileName(1, 3), 1));
                // after the save, just the one "begin"
                NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
                                                                  GetInProgressEditsFileName(4), 4));
            }
            finally
            {
                Log.Info("Closing nn");
                if (namesystem != null)
                {
                    namesystem.Close();
                }
            }
        }
Exemplo n.º 2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestResolveReservedPath()
        {
            FSNamesystem fsn     = SetupFileSystem();
            FSEditLog    editlog = fsn.GetEditLog();

            fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
            Org.Mockito.Mockito.Verify(editlog).LogTimes(Matchers.Eq(FilePath), Matchers.AnyLong
                                                             (), Matchers.AnyLong());
            fsn.Close();
        }
Exemplo n.º 3
0
        public virtual void TestSaveRightBeforeSync()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);

            try
            {
                FSImage   fsimage = namesystem.GetFSImage();
                FSEditLog editLog = Org.Mockito.Mockito.Spy(fsimage.GetEditLog());
                DFSTestUtil.SetEditLogForTesting(namesystem, editLog);
                AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
                CountDownLatch waitToEnterSync = new CountDownLatch(1);
                Sharpen.Thread doAnEditThread  = new _Thread_467(namesystem, deferredException, waitToEnterSync
                                                                 );
                Answer <Void> blockingSync = new _Answer_484(doAnEditThread, waitToEnterSync);
                Org.Mockito.Mockito.DoAnswer(blockingSync).When(editLog).LogSync();
                doAnEditThread.Start();
                Log.Info("Main thread: waiting to just before logSync...");
                waitToEnterSync.Await();
                NUnit.Framework.Assert.IsNull(deferredException.Get());
                Log.Info("Main thread: detected that logSync about to be called.");
                Log.Info("Trying to enter safe mode.");
                Log.Info("This should block for " + BlockTime + "sec, since we have pending edits"
                         );
                long st = Time.Now();
                namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                long et = Time.Now();
                Log.Info("Entered safe mode");
                // Make sure we really waited for the flush to complete!
                NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
                // Once we're in safe mode, save namespace.
                namesystem.SaveNamespace();
                Log.Info("Joining on edit thread...");
                doAnEditThread.Join();
                NUnit.Framework.Assert.IsNull(deferredException.Get());
                // We did 3 edits: begin, txn, and end
                NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
                                                                  GetFinalizedEditsFileName(1, 3), 1));
                // after the save, just the one "begin"
                NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
                                                                  GetInProgressEditsFileName(4), 4));
            }
            finally
            {
                Log.Info("Closing nn");
                if (namesystem != null)
                {
                    namesystem.Close();
                }
            }
        }
Exemplo n.º 4
0
 public virtual void TestRetryCacheMetrics()
 {
     CheckMetrics(0, 0, 0);
     // DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY is 2 ,
     // so 2 requests are dropped at first.
     // After that, 1 request will reach NameNode correctly.
     TrySaveNamespace();
     CheckMetrics(2, 0, 1);
     // RetryCache will be cleared after Namesystem#close()
     namesystem.Close();
     CheckMetrics(2, 1, 1);
 }
Exemplo n.º 5
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestGetBlockLocationsRacingWithDelete()
        {
            FSNamesystem fsn     = Org.Mockito.Mockito.Spy(SetupFileSystem());
            FSDirectory  fsd     = fsn.GetFSDirectory();
            FSEditLog    editlog = fsn.GetEditLog();

            Org.Mockito.Mockito.DoAnswer(new _Answer_67(fsd)).When(fsn).WriteLock();
            fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
            Org.Mockito.Mockito.Verify(editlog, Org.Mockito.Mockito.Never()).LogTimes(Matchers.AnyString
                                                                                          (), Matchers.AnyLong(), Matchers.AnyLong());
            fsn.Close();
        }
Exemplo n.º 6
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestGetBlockLocationsRacingWithRename()
        {
            FSNamesystem fsn     = Org.Mockito.Mockito.Spy(SetupFileSystem());
            FSDirectory  fsd     = fsn.GetFSDirectory();
            FSEditLog    editlog = fsn.GetEditLog();
            string       DstPath = "/bar";

            bool[] renamed = new bool[1];
            Org.Mockito.Mockito.DoAnswer(new _Answer_92(renamed, fsd, DstPath)).When(fsn).WriteLock
                ();
            fsn.GetBlockLocations("dummy", ReservedPath, 0, 1024);
            Org.Mockito.Mockito.Verify(editlog).LogTimes(Matchers.Eq(DstPath), Matchers.AnyLong
                                                             (), Matchers.AnyLong());
            fsn.Close();
        }
Exemplo n.º 7
0
        /// <exception cref="System.Exception"/>
        public virtual void TestTxIdPersistence()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);

            try
            {
                // We have a BEGIN_LOG_SEGMENT txn to start
                NUnit.Framework.Assert.AreEqual(1, fsn.GetEditLog().GetLastWrittenTxId());
                DoAnEdit(fsn, 1);
                NUnit.Framework.Assert.AreEqual(2, fsn.GetEditLog().GetLastWrittenTxId());
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                fsn.SaveNamespace();
                // 2 more txns: END the first segment, BEGIN a new one
                NUnit.Framework.Assert.AreEqual(4, fsn.GetEditLog().GetLastWrittenTxId());
                // Shut down and restart
                fsn.GetFSImage().Close();
                fsn.Close();
                // 1 more txn to END that segment
                NUnit.Framework.Assert.AreEqual(5, fsn.GetEditLog().GetLastWrittenTxId());
                fsn = null;
                fsn = FSNamesystem.LoadFromDisk(conf);
                // 1 more txn to start new segment on restart
                NUnit.Framework.Assert.AreEqual(6, fsn.GetEditLog().GetLastWrittenTxId());
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
Exemplo n.º 8
0
        /// <exception cref="System.Exception"/>
        public virtual void TestSaveWhileEditsRolled()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);

            try
            {
                DoAnEdit(fsn, 1);
                CheckpointSignature sig = fsn.RollEditLog();
                Log.Warn("Checkpoint signature: " + sig);
                // Do another edit
                DoAnEdit(fsn, 2);
                // Save namespace
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                fsn.SaveNamespace();
                // Now shut down and restart the NN
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edits.
                CheckEditExists(fsn, 1);
                CheckEditExists(fsn, 2);
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
Exemplo n.º 9
0
        /// <exception cref="System.Exception"/>
        private void SaveNamespaceWithInjectedFault(TestSaveNamespace.Fault fault)
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();
            NNStorage spyStorage    = Org.Mockito.Mockito.Spy(storage);

            originalImage.storage = spyStorage;
            FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            bool shouldFail = false;

            switch (fault)
            {
            case TestSaveNamespace.Fault.SaveSecondFsimageRte:
            {
                // should we expect the save operation to fail
                // inject fault
                // The spy throws a RuntimeException when writing to the second directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyImage
                                                                                               ).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                                                                                          )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = false;
                break;
            }

            case TestSaveNamespace.Fault.SaveSecondFsimageIoe:
            {
                // The spy throws an IOException when writing to the second directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(false)).When(spyImage
                                                                                                ).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                                                                                           )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = false;
                break;
            }

            case TestSaveNamespace.Fault.SaveAllFsimages:
            {
                // The spy throws IOException in all directories
                Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyImage).SaveFSImage
                    ((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory)Matchers.AnyObject
                        (), (NNStorage.NameNodeFile)Matchers.AnyObject());
                shouldFail = true;
                break;
            }

            case TestSaveNamespace.Fault.WriteStorageAll:
            {
                // The spy throws an exception before writing any VERSION files
                Org.Mockito.Mockito.DoThrow(new RuntimeException("Injected")).When(spyStorage).WriteAll
                    ();
                shouldFail = true;
                break;
            }

            case TestSaveNamespace.Fault.WriteStorageOne:
            {
                // The spy throws on exception on one particular storage directory
                Org.Mockito.Mockito.DoAnswer(new TestSaveNamespace.FaultySaveImage(true)).When(spyStorage
                                                                                               ).WriteProperties((Storage.StorageDirectory)Matchers.AnyObject());
                // TODO: unfortunately this fails -- should be improved.
                // See HDFS-2173.
                shouldFail = true;
                break;
            }
            }
            try
            {
                DoAnEdit(fsn, 1);
                // Save namespace - this may fail, depending on fault injected
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    fsn.SaveNamespace();
                    if (shouldFail)
                    {
                        NUnit.Framework.Assert.Fail("Did not fail!");
                    }
                }
                catch (Exception e)
                {
                    if (!shouldFail)
                    {
                        throw;
                    }
                    else
                    {
                        Log.Info("Test caught expected exception", e);
                    }
                }
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
                // Should still be able to perform edits
                DoAnEdit(fsn, 2);
                // Now shut down and restart the namesystem
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edits.
                CheckEditExists(fsn, 1);
                CheckEditExists(fsn, 2);
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
Exemplo n.º 10
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCancelSaveNamespace()
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   image   = fsn.GetFSImage();
            NNStorage storage = image.GetStorage();

            storage.Close();
            // unlock any directories that FSNamesystem's initialization may have locked
            storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem.GetNamespaceEditsDirs
                                              (conf));
            FSNamesystem spyFsn   = Org.Mockito.Mockito.Spy(fsn);
            FSNamesystem finalFsn = spyFsn;

            GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
            BlockIdManager bid = Org.Mockito.Mockito.Spy(spyFsn.GetBlockIdManager());

            Whitebox.SetInternalState(finalFsn, "blockIdManager", bid);
            Org.Mockito.Mockito.DoAnswer(delayer).When(bid).GetGenerationStampV2();
            ExecutorService pool = Executors.NewFixedThreadPool(2);

            try
            {
                DoAnEdit(fsn, 1);
                Canceler canceler = new Canceler();
                // Save namespace
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    Future <Void> saverFuture = pool.Submit(new _Callable_561(image, finalFsn, canceler
                                                                              ));
                    // Wait until saveNamespace calls getGenerationStamp
                    delayer.WaitForCall();
                    // then cancel the saveNamespace
                    Future <Void> cancelFuture = pool.Submit(new _Callable_572(canceler));
                    // give the cancel call time to run
                    Sharpen.Thread.Sleep(500);
                    // allow saveNamespace to proceed - it should check the cancel flag after
                    // this point and throw an exception
                    delayer.Proceed();
                    cancelFuture.Get();
                    saverFuture.Get();
                    NUnit.Framework.Assert.Fail("saveNamespace did not fail even though cancelled!");
                }
                catch (Exception t)
                {
                    GenericTestUtils.AssertExceptionContains("SaveNamespaceCancelledException", t);
                }
                Log.Info("Successfully cancelled a saveNamespace");
                // Check that we have only the original image and not any
                // cruft left over from half-finished images
                FSImageTestUtil.LogStorageContents(Log, storage);
                foreach (Storage.StorageDirectory sd in storage.DirIterable(null))
                {
                    FilePath curDir = sd.GetCurrentDir();
                    GenericTestUtils.AssertGlobEquals(curDir, "fsimage_.*", NNStorage.GetImageFileName
                                                          (0), NNStorage.GetImageFileName(0) + MD5FileUtils.Md5Suffix);
                }
            }
            finally
            {
                fsn.Close();
            }
        }
Exemplo n.º 11
0
        /// <summary>Injects a failure on all storage directories while saving namespace.</summary>
        /// <param name="restoreStorageAfterFailure">
        /// if true, will try to save again after
        /// clearing the failure injection
        /// </param>
        /// <exception cref="System.Exception"/>
        public virtual void DoTestFailedSaveNamespace(bool restoreStorageAfterFailure)
        {
            Configuration conf = GetConf();

            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();

            storage.Close();
            // unlock any directories that FSNamesystem's initialization may have locked
            NNStorage spyStorage = Org.Mockito.Mockito.Spy(storage);

            originalImage.storage = spyStorage;
            FSImage spyImage = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            spyImage.storage.SetStorageDirectories(FSNamesystem.GetNamespaceDirs(conf), FSNamesystem
                                                   .GetNamespaceEditsDirs(conf));
            Org.Mockito.Mockito.DoThrow(new IOException("Injected fault: saveFSImage")).When(
                spyImage).SaveFSImage((SaveNamespaceContext)Matchers.AnyObject(), (Storage.StorageDirectory
                                                                                   )Matchers.AnyObject(), (NNStorage.NameNodeFile)Matchers.AnyObject());
            try
            {
                DoAnEdit(fsn, 1);
                // Save namespace
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                try
                {
                    fsn.SaveNamespace();
                    NUnit.Framework.Assert.Fail("saveNamespace did not fail even when all directories failed!"
                                                );
                }
                catch (IOException ioe)
                {
                    Log.Info("Got expected exception", ioe);
                }
                // Ensure that, if storage dirs come back online, things work again.
                if (restoreStorageAfterFailure)
                {
                    Org.Mockito.Mockito.Reset(spyImage);
                    spyStorage.SetRestoreFailedStorage(true);
                    fsn.SaveNamespace();
                    CheckEditExists(fsn, 1);
                }
                // Now shut down and restart the NN
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edits.
                CheckEditExists(fsn, 1);
            }
            finally
            {
                if (fsn != null)
                {
                    fsn.Close();
                }
            }
        }
Exemplo n.º 12
0
        /// <summary>
        /// Verify that a saveNamespace command brings faulty directories
        /// in fs.name.dir and fs.edit.dir back online.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestReinsertnamedirsInSavenamespace()
        {
            // create a configuration with the key to restore error
            // directories in fs.name.dir
            Configuration conf = GetConf();

            conf.SetBoolean(DFSConfigKeys.DfsNamenodeNameDirRestoreKey, true);
            NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
            DFSTestUtil.FormatNameNode(conf);
            FSNamesystem fsn = FSNamesystem.LoadFromDisk(conf);
            // Replace the FSImage with a spy
            FSImage   originalImage = fsn.GetFSImage();
            NNStorage storage       = originalImage.GetStorage();
            FSImage   spyImage      = Org.Mockito.Mockito.Spy(originalImage);

            Whitebox.SetInternalState(fsn, "fsImage", spyImage);
            FileSystem   fs             = FileSystem.GetLocal(conf);
            FilePath     rootDir        = storage.GetStorageDir(0).GetRoot();
            Path         rootPath       = new Path(rootDir.GetPath(), "current");
            FsPermission permissionNone = new FsPermission((short)0);
            FsPermission permissionAll  = new FsPermission(FsAction.All, FsAction.ReadExecute,
                                                           FsAction.ReadExecute);

            fs.SetPermission(rootPath, permissionNone);
            try
            {
                DoAnEdit(fsn, 1);
                fsn.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
                // Save namespace - should mark the first storage dir as faulty
                // since it's not traversable.
                Log.Info("Doing the first savenamespace.");
                fsn.SaveNamespace();
                Log.Info("First savenamespace sucessful.");
                NUnit.Framework.Assert.IsTrue("Savenamespace should have marked one directory as bad."
                                              + " But found " + storage.GetRemovedStorageDirs().Count + " bad directories.",
                                              storage.GetRemovedStorageDirs().Count == 1);
                fs.SetPermission(rootPath, permissionAll);
                // The next call to savenamespace should try inserting the
                // erroneous directory back to fs.name.dir. This command should
                // be successful.
                Log.Info("Doing the second savenamespace.");
                fsn.SaveNamespace();
                Log.Warn("Second savenamespace sucessful.");
                NUnit.Framework.Assert.IsTrue("Savenamespace should have been successful in removing "
                                              + " bad directories from Image." + " But found " + storage.GetRemovedStorageDirs
                                                  ().Count + " bad directories.", storage.GetRemovedStorageDirs().Count == 0);
                // Now shut down and restart the namesystem
                Log.Info("Shutting down fsimage.");
                originalImage.Close();
                fsn.Close();
                fsn = null;
                // Start a new namesystem, which should be able to recover
                // the namespace from the previous incarnation.
                Log.Info("Loading new FSmage from disk.");
                fsn = FSNamesystem.LoadFromDisk(conf);
                // Make sure the image loaded including our edit.
                Log.Info("Checking reloaded image.");
                CheckEditExists(fsn, 1);
                Log.Info("Reloaded image is good.");
            }
            finally
            {
                if (rootDir.Exists())
                {
                    fs.SetPermission(rootPath, permissionAll);
                }
                if (fsn != null)
                {
                    try
                    {
                        fsn.Close();
                    }
                    catch (Exception t)
                    {
                        Log.Fatal("Failed to shut down", t);
                    }
                }
            }
        }