public virtual void TestEmptyDir() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); RunTest(tc); }
public virtual void TestNoLogs() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); RunTest(tc); }
public virtual void TestPurgeLessThanRetention() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), false); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), false ); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false ); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false ); tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false); RunTest(tc); }
public virtual void TestPurgeEasyCase() { TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true); tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false ); tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false); // Test that other files don't get purged tc.AddLog("/foo1/current/VERSION", false); RunTest(tc); }
public virtual void TestRetainExtraLogs() { conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50); TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image); tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true); // Since we need 50 extra edits, *do* retain the 201-300 segment tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false ); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false); RunTest(tc); }
public virtual void TestRetainExtraLogsLimitedSegments() { conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150); conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2); TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription (this); tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image); tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false); tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false); // Segments containing txns upto txId 250 are extra and should be purged. tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty" , true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt" , true); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true); // Only retain 2 extra segments. The 301-350 and 351-400 segments are // considered required, not extra. tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false ); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty" , false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt" , false); tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false ); tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false); RunTest(tc); }
/// <exception cref="System.IO.IOException"/> private void RunTest(TestNNStorageRetentionManager.TestCaseDescription tc) { NNStorageRetentionManager.StoragePurger mockPurger = Org.Mockito.Mockito.Mock <NNStorageRetentionManager.StoragePurger >(); ArgumentCaptor <FSImageStorageInspector.FSImageFile> imagesPurgedCaptor = ArgumentCaptor .ForClass <FSImageStorageInspector.FSImageFile>(); ArgumentCaptor <FileJournalManager.EditLogFile> logsPurgedCaptor = ArgumentCaptor. ForClass <FileJournalManager.EditLogFile>(); // Ask the manager to purge files we don't need any more new NNStorageRetentionManager(conf, tc.MockStorage(), tc.MockEditLog(mockPurger), mockPurger).PurgeOldStorage(NNStorage.NameNodeFile.Image); // Verify that it asked the purger to remove the correct files Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeImage (imagesPurgedCaptor.Capture()); Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeLog(logsPurgedCaptor .Capture()); // Check images ICollection <string> purgedPaths = Sets.NewLinkedHashSet(); foreach (FSImageStorageInspector.FSImageFile purged in imagesPurgedCaptor.GetAllValues ()) { purgedPaths.AddItem(FileToPath(purged.GetFile())); } NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedImages )), Joiner.On(",").Join(purgedPaths)); // Check images purgedPaths.Clear(); foreach (FileJournalManager.EditLogFile purged_1 in logsPurgedCaptor.GetAllValues ()) { purgedPaths.AddItem(FileToPath(purged_1.GetFile())); } NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedLogs )), Joiner.On(",").Join(purgedPaths)); }