public virtual void TestExcludeInProgressStreams()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams");
// Don't close the edit log once the files have been set up.
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, false);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// If we exclude the in-progess stream, we should only have 100 tx.
NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false)
);
EditLogInputStream elis = GetJournalInputStream(jm, 90, false);
try
{
FSEditLogOp lastReadOp = null;
while ((lastReadOp = elis.ReadOp()) != null)
{
NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100);
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
public virtual void TestDoPreUpgradeIOError()
{
FilePath storageDir = new FilePath(TestEditLog.TestDir, "preupgradeioerror");
IList <URI> editUris = Sharpen.Collections.SingletonList(storageDir.ToURI());
NNStorage storage = TestEditLog.SetupEdits(editUris, 5);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
NUnit.Framework.Assert.IsNotNull(sd);
// Change storage directory so that renaming current to previous.tmp fails.
FileUtil.SetWritable(storageDir, false);
FileJournalManager jm = null;
try
{
jm = new FileJournalManager(conf, sd, storage);
exception.Expect(typeof(IOException));
if (NativeCodeLoader.IsNativeCodeLoaded())
{
exception.ExpectMessage("failure in native rename");
}
jm.DoPreUpgrade();
}
finally
{
IOUtils.Cleanup(Log, jm);
// Restore permissions on storage directory and make sure we can delete.
FileUtil.SetWritable(storageDir, true);
FileUtil.FullyDelete(storageDir);
}
}
public virtual void TestInprogressRecoveryAll()
{
FilePath f1 = new FilePath(TestEditLog.TestDir + "/failalltest0");
FilePath f2 = new FilePath(TestEditLog.TestDir + "/failalltest1");
FilePath f3 = new FilePath(TestEditLog.TestDir + "/failalltest2");
IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI());
// abort after the 5th roll
NNStorage storage = TestEditLog.SetupEdits(editUris, 5, new TestEditLog.AbortSpec
(5, 0), new TestEditLog.AbortSpec(5, 1), new TestEditLog.AbortSpec(5, 2));
IEnumerator <Storage.StorageDirectory> dirs = storage.DirIterator(NNStorage.NameNodeDirType
.Edits);
Storage.StorageDirectory sd = dirs.Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
sd = dirs.Next();
jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
sd = dirs.Next();
jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
}
public virtual void TestManyLogsWithGaps()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/manylogswithgaps");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
long startGapTxId = 3 * TestEditLog.TxnsPerRoll + 1;
long endGapTxId = 4 * TestEditLog.TxnsPerRoll;
FilePath[] files = new FilePath(f, "current").ListFiles(new _FilenameFilter_324(startGapTxId
, endGapTxId));
NUnit.Framework.Assert.AreEqual(1, files.Length);
NUnit.Framework.Assert.IsTrue(files[0].Delete());
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(startGapTxId - 1, GetNumberOfTransactions(jm, 1,
true, true));
NUnit.Framework.Assert.AreEqual(0, GetNumberOfTransactions(jm, startGapTxId, true
, true));
// rolled 10 times so there should be 11 files.
NUnit.Framework.Assert.AreEqual(11 * TestEditLog.TxnsPerRoll - endGapTxId, GetNumberOfTransactions
(jm, endGapTxId + 1, true, true));
}
public virtual void TestInprogressRecovery()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/inprogressrecovery");
// abort after the 5th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 5, new TestEditLog.AbortSpec(5, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(5 * TestEditLog.TxnsPerRoll + TestEditLog.TxnsPerFail
, GetNumberOfTransactions(jm, 1, true, false));
}
public virtual void TestManyLogsWithCorruptInprogress()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/manylogswithcorruptinprogress");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FilePath[] files = new FilePath(f, "current").ListFiles(new _FilenameFilter_356()
);
NUnit.Framework.Assert.AreEqual(files.Length, 1);
CorruptAfterStartSegment(files[0]);
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(10 * TestEditLog.TxnsPerRoll + 1, GetNumberOfTransactions
(jm, 1, true, false));
}
public virtual void TestAskForTransactionsMidfile()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/askfortransactionsmidfile");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// 10 rolls, so 11 rolled files, 110 txids total.
int TotalTxids = 10 * 11;
for (int txid = 1; txid <= TotalTxids; txid++)
{
NUnit.Framework.Assert.AreEqual((TotalTxids - txid) + 1, GetNumberOfTransactions(
jm, txid, true, false));
}
}
public virtual void TestNormalOperation()
{
FilePath f1 = new FilePath(TestEditLog.TestDir + "/normtest0");
FilePath f2 = new FilePath(TestEditLog.TestDir + "/normtest1");
FilePath f3 = new FilePath(TestEditLog.TestDir + "/normtest2");
IList <URI> editUris = ImmutableList.Of(f1.ToURI(), f2.ToURI(), f3.ToURI());
NNStorage storage = TestEditLog.SetupEdits(editUris, 5);
long numJournals = 0;
foreach (Storage.StorageDirectory sd in storage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
NUnit.Framework.Assert.AreEqual(6 * TestEditLog.TxnsPerRoll, GetNumberOfTransactions
(jm, 1, true, false));
numJournals++;
}
NUnit.Framework.Assert.AreEqual(3, numJournals);
}
public virtual void TestReadFromMiddleOfEditLog()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
EditLogInputStream elis = GetJournalInputStream(jm, 5, true);
try
{
FSEditLogOp op = elis.ReadOp();
NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5);
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
public virtual void TestReadFromStream()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfromstream");
// abort after 10th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
long expectedTotalTxnCount = TestEditLog.TxnsPerRoll * 10 + TestEditLog.TxnsPerFail;
NUnit.Framework.Assert.AreEqual(expectedTotalTxnCount, GetNumberOfTransactions(jm
, 1, true, false));
long skippedTxns = (3 * TestEditLog.TxnsPerRoll);
// skip first 3 files
long startingTxId = skippedTxns + 1;
long numLoadable = GetNumberOfTransactions(jm, startingTxId, true, false);
NUnit.Framework.Assert.AreEqual(expectedTotalTxnCount - skippedTxns, numLoadable);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestFinalizeErrorReportedToNNStorage()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/filejournaltestError");
// abort after 10th roll
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, new TestEditLog.AbortSpec(10, 0));
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
string sdRootPath = sd.GetRoot().GetAbsolutePath();
FileUtil.Chmod(sdRootPath, "-w", true);
try
{
jm.FinalizeLogSegment(0, 1);
}
finally
{
FileUtil.Chmod(sdRootPath, "+w", true);
NUnit.Framework.Assert.IsTrue(storage.GetRemovedStorageDirs().Contains(sd));
}
}