public virtual void TestExcludeInProgressStreams() { FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10, false); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false) ); EditLogInputStream elis = GetJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.ReadOp()) != null) { NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100); } } finally { IOUtils.Cleanup(Log, elis); } }
/// <summary> /// Find out how many transactions we can read from a /// FileJournalManager, starting at a given transaction ID. /// </summary> /// <param name="jm">The journal manager</param> /// <param name="fromTxId">Transaction ID to start at</param> /// <param name="inProgressOk">Should we consider edit logs that are not finalized?</param> /// <returns>The number of transactions</returns> /// <exception cref="System.IO.IOException"/> internal static long GetNumberOfTransactions(FileJournalManager jm, long fromTxId , bool inProgressOk, bool abortOnGap) { long numTransactions = 0; long txId = fromTxId; PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream >(64, JournalSet.EditLogInputStreamComparator); jm.SelectInputStreams(allStreams, fromTxId, inProgressOk); EditLogInputStream elis = null; try { while ((elis = allStreams.Poll()) != null) { try { elis.SkipUntil(txId); while (true) { FSEditLogOp op = elis.ReadOp(); if (op == null) { break; } if (abortOnGap && (op.GetTransactionId() != txId)) { Log.Info("getNumberOfTransactions: detected gap at txId " + fromTxId); return(numTransactions); } txId = op.GetTransactionId() + 1; numTransactions++; } } finally { IOUtils.Cleanup(Log, elis); } } } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream [0])); } return(numTransactions); }
/// <seealso cref="CountEditLogOpTypes(Sharpen.FilePath)"/> /// <exception cref="System.IO.IOException"/> public static EnumMap <FSEditLogOpCodes, Holder <int> > CountEditLogOpTypes(EditLogInputStream elis) { EnumMap <FSEditLogOpCodes, Holder <int> > opCounts = new EnumMap <FSEditLogOpCodes, Holder <int> >(typeof(FSEditLogOpCodes)); FSEditLogOp op; while ((op = elis.ReadOp()) != null) { Holder <int> i = opCounts[op.opCode]; if (i == null) { i = new Holder <int>(0); opCounts[op.opCode] = i; } i.held++; } return(opCounts); }
public virtual void TestReadFromMiddleOfEditLog() { FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog"); NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); EditLogInputStream elis = GetJournalInputStream(jm, 5, true); try { FSEditLogOp op = elis.ReadOp(); NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5); } finally { IOUtils.Cleanup(Log, elis); } }