public virtual void TestEmptyDir()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
RunTest(tc);
}
public virtual void TestNoLogs()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
RunTest(tc);
}
public virtual void TestPurgeLessThanRetention()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), false);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), false
);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
);
tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
RunTest(tc);
}
public virtual void TestPurgeEasyCase()
{
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.ImageAndEdits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), true);
tc.AddLog("/foo1/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
);
tc.AddLog("/foo1/current/" + NNStorage.GetInProgressEditsFileName(401), false);
// Test that other files don't get purged
tc.AddLog("/foo1/current/VERSION", false);
RunTest(tc);
}
public virtual void TestRetainExtraLogs()
{
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 50);
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 200), true);
// Since we need 50 extra edits, *do* retain the 201-300 segment
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 300), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 400), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
RunTest(tc);
}
public virtual void TestRetainExtraLogsLimitedSegments()
{
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 150);
conf.SetLong(DFSConfigKeys.DfsNamenodeMaxExtraEditsSegmentsRetainedKey, 2);
TestNNStorageRetentionManager.TestCaseDescription tc = new TestNNStorageRetentionManager.TestCaseDescription
(this);
tc.AddRoot("/foo1", NNStorage.NameNodeDirType.Image);
tc.AddRoot("/foo2", NNStorage.NameNodeDirType.Edits);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(100), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(200), true);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(300), false);
tc.AddImage("/foo1/current/" + NNStorage.GetImageFileName(400), false);
// Segments containing txns upto txId 250 are extra and should be purged.
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(1, 100), true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(101, 175), true);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(176) + ".empty"
, true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(176, 200), true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(201, 225), true);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(226) + ".corrupt"
, true);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(226, 240), true);
// Only retain 2 extra segments. The 301-350 and 351-400 segments are
// considered required, not extra.
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(241, 275), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(276, 300), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(301) + ".empty"
, false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(301, 350), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(351) + ".corrupt"
, false);
tc.AddLog("/foo2/current/" + NNStorage.GetFinalizedEditsFileName(351, 400), false
);
tc.AddLog("/foo2/current/" + NNStorage.GetInProgressEditsFileName(401), false);
RunTest(tc);
}
/// <exception cref="System.IO.IOException"/>
private void RunTest(TestNNStorageRetentionManager.TestCaseDescription tc)
{
NNStorageRetentionManager.StoragePurger mockPurger = Org.Mockito.Mockito.Mock <NNStorageRetentionManager.StoragePurger
>();
ArgumentCaptor <FSImageStorageInspector.FSImageFile> imagesPurgedCaptor = ArgumentCaptor
.ForClass <FSImageStorageInspector.FSImageFile>();
ArgumentCaptor <FileJournalManager.EditLogFile> logsPurgedCaptor = ArgumentCaptor.
ForClass <FileJournalManager.EditLogFile>();
// Ask the manager to purge files we don't need any more
new NNStorageRetentionManager(conf, tc.MockStorage(), tc.MockEditLog(mockPurger),
mockPurger).PurgeOldStorage(NNStorage.NameNodeFile.Image);
// Verify that it asked the purger to remove the correct files
Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeImage
(imagesPurgedCaptor.Capture());
Org.Mockito.Mockito.Verify(mockPurger, Org.Mockito.Mockito.AtLeast(0)).PurgeLog(logsPurgedCaptor
.Capture());
// Check images
ICollection <string> purgedPaths = Sets.NewLinkedHashSet();
foreach (FSImageStorageInspector.FSImageFile purged in imagesPurgedCaptor.GetAllValues
())
{
purgedPaths.AddItem(FileToPath(purged.GetFile()));
}
NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedImages
)), Joiner.On(",").Join(purgedPaths));
// Check images
purgedPaths.Clear();
foreach (FileJournalManager.EditLogFile purged_1 in logsPurgedCaptor.GetAllValues
())
{
purgedPaths.AddItem(FileToPath(purged_1.GetFile()));
}
NUnit.Framework.Assert.AreEqual(Joiner.On(",").Join(FilesToPaths(tc.expectedPurgedLogs
)), Joiner.On(",").Join(purgedPaths));
}