public virtual void TestEmptyDir()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     RunTest(tc);
 }
 public virtual void TestNoLogs()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     RunTest(tc);
 }
 public virtual void TestPurgeLessThanRetention()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), false);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
 public virtual void TestPurgeEasyCase()
 {
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true);
     tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     // Test that other files don't get purged
     tc.AddLog("/foo1/current/VERSION", false);
     RunTest(tc);
 }
 public virtual void TestRetainExtraLogs()
 {
     conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50);
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
     tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
     // Since we need 50 extra edits, *do* retain the 201-300 segment
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
 public virtual void TestRetainExtraLogsLimitedSegments()
 {
     conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150);
     conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2);
     TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
                                                                (this);
     tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
     tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
     tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
     // Segments containing txns upto txId 250 are extra and should be purged.
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty"
               , true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true);
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt"
               , true);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true);
     // Only retain 2 extra segments. The 301-350 and 351-400 segments are
     // considered required, not extra.
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty"
               , false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt"
               , false);
     tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false
               );
     tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
     RunTest(tc);
 }
        /// <exception cref="System.IO.IOException"/>
        private void RunTest(TestNNStorageRetentionManager.TestCaseDescription tc)
        {
            NNStorageRetentionManager.StoragePurger mockPurger = Org.Mockito.Mockito.Mock <NNStorageRetentionManager.StoragePurger
                                                                                           >();
            ArgumentCaptor <FSImageStorageInspector.FSImageFile> imagesPurgedCaptor = ArgumentCaptor
                                                                                      .ForClass <FSImageStorageInspector.FSImageFile>();
            ArgumentCaptor <FileJournalManager.EditLogFile> logsPurgedCaptor = ArgumentCaptor.
                                                                               ForClass <FileJournalManager.EditLogFile>();

            // Ask the manager to purge files we don't need any more
            new NNStorageRetentionManager(conf, tc.MockStorage(), tc.MockEditLog(mockPurger),
                                          mockPurger).PurgeOldStorage(NNStorage.NameNodeFile.Image);
            // Verify that it asked the purger to remove the correct files
            Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeImage
                (imagesPurgedCaptor.Capture());
            Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeLog(logsPurgedCaptor
                                                                                            .Capture());
            // Check images
            ICollection <string> purgedPaths = Sets.NewLinkedHashSet();

            foreach (FSImageStorageInspector.FSImageFile purged in imagesPurgedCaptor.GetAllValues
                         ())
            {
                purgedPaths.AddItem(FileToPath(purged.GetFile()));
            }
            NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedImages
                                                                             )), Joiner.On(",").Join(purgedPaths));
            // Check images
            purgedPaths.Clear();
            foreach (FileJournalManager.EditLogFile purged_1 in logsPurgedCaptor.GetAllValues
                         ())
            {
                purgedPaths.AddItem(FileToPath(purged_1.GetFile()));
            }
            NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedLogs
                                                                             )), Joiner.On(",").Join(purgedPaths));
        }