internal RedundantEditLogInputStream(ICollection <EditLogInputStream> streams, long startTxId) { this.curIdx = 0; this.prevTxId = (startTxId == HdfsConstants.InvalidTxid) ? HdfsConstants.InvalidTxid : (startTxId - 1); this.state = (streams.IsEmpty()) ? RedundantEditLogInputStream.State.Eof : RedundantEditLogInputStream.State .SkipUntil; this.prevException = null; // EditLogInputStreams in a RedundantEditLogInputStream must be finalized, // and can't be pre-transactional. EditLogInputStream first = null; foreach (EditLogInputStream s in streams) { Preconditions.CheckArgument(s.GetFirstTxId() != HdfsConstants.InvalidTxid, "invalid first txid in stream: %s" , s); Preconditions.CheckArgument(s.GetLastTxId() != HdfsConstants.InvalidTxid, "invalid last txid in stream: %s" , s); if (first == null) { first = s; } else { Preconditions.CheckArgument(s.GetFirstTxId() == first.GetFirstTxId(), "All streams in the RedundantEditLogInputStream must have the same " + "start transaction ID! " + first + " had start txId " + first.GetFirstTxId() + ", but " + s + " had start txId " + s.GetFirstTxId()); } } this.streams = Sharpen.Collections.ToArray(streams, new EditLogInputStream[0]); // We sort the streams here so that the streams that end later come first. Arrays.Sort(this.streams, new _IComparer_117()); }
public virtual void TestReadURL() { HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>(); Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream (); Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode (); Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When (conn).GetHeaderField("Content-Length"); URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>(); Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito .Any <Uri>(), Matchers.AnyBoolean()); Uri url = new Uri("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants .InvalidTxid, HdfsConstants.InvalidTxid, false); // Read the edit log and verify that we got all of the data. EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes (elis); Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1)); Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is( 1)); Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1)); // Check that length header was picked up. NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length()); elis.Close(); }
public virtual void TestExcludeInProgressStreams() { FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10, false); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false) ); EditLogInputStream elis = GetJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.ReadOp()) != null) { NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100); } } finally { IOUtils.Cleanup(Log, elis); } }
/// <summary> /// Find out how many transactions we can read from a /// FileJournalManager, starting at a given transaction ID. /// </summary> /// <param name="jm">The journal manager</param> /// <param name="fromTxId">Transaction ID to start at</param> /// <param name="inProgressOk">Should we consider edit logs that are not finalized?</param> /// <returns>The number of transactions</returns> /// <exception cref="System.IO.IOException"/> internal static long GetNumberOfTransactions(FileJournalManager jm, long fromTxId , bool inProgressOk, bool abortOnGap) { long numTransactions = 0; long txId = fromTxId; PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream >(64, JournalSet.EditLogInputStreamComparator); jm.SelectInputStreams(allStreams, fromTxId, inProgressOk); EditLogInputStream elis = null; try { while ((elis = allStreams.Poll()) != null) { try { elis.SkipUntil(txId); while (true) { FSEditLogOp op = elis.ReadOp(); if (op == null) { break; } if (abortOnGap && (op.GetTransactionId() != txId)) { Log.Info("getNumberOfTransactions: detected gap at txId " + fromTxId); return(numTransactions); } txId = op.GetTransactionId() + 1; numTransactions++; } } finally { IOUtils.Cleanup(Log, elis); } } } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream [0])); } return(numTransactions); }
/// <seealso cref="CountEditLogOpTypes(Sharpen.FilePath)"/> /// <exception cref="System.IO.IOException"/> public static EnumMap <FSEditLogOpCodes, Holder <int> > CountEditLogOpTypes(EditLogInputStream elis) { EnumMap <FSEditLogOpCodes, Holder <int> > opCounts = new EnumMap <FSEditLogOpCodes, Holder <int> >(typeof(FSEditLogOpCodes)); FSEditLogOp op; while ((op = elis.ReadOp()) != null) { Holder <int> i = opCounts[op.opCode]; if (i == null) { i = new Holder <int>(0); opCounts[op.opCode] = i; } i.held++; } return(opCounts); }
public virtual void TestReadFromMiddleOfEditLog() { FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog"); NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI> (f.ToURI()), 10); Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits ).Next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); EditLogInputStream elis = GetJournalInputStream(jm, 5, true); try { FSEditLogOp op = elis.ReadOp(); NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5); } finally { IOUtils.Cleanup(Log, elis); } }
/// <exception cref="System.IO.IOException"/> private static EditLogInputStream GetJournalInputStream(JournalManager jm, long txId , bool inProgressOk) { PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream >(64, JournalSet.EditLogInputStreamComparator); jm.SelectInputStreams(allStreams, txId, inProgressOk); EditLogInputStream elis = null; EditLogInputStream ret; try { while ((elis = allStreams.Poll()) != null) { if (elis.GetFirstTxId() > txId) { break; } if (elis.GetLastTxId() < txId) { elis.Close(); continue; } elis.SkipUntil(txId); ret = elis; elis = null; return(ret); } } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream [0])); IOUtils.Cleanup(Log, elis); } return(null); }
public static void ChainAndMakeRedundantStreams(ICollection <EditLogInputStream> outStreams , PriorityQueue <EditLogInputStream> allStreams, long fromTxId) { // We want to group together all the streams that start on the same start // transaction ID. To do this, we maintain an accumulator (acc) of all // the streams we've seen at a given start transaction ID. When we see a // higher start transaction ID, we select a stream from the accumulator and // clear it. Then we begin accumulating streams with the new, higher start // transaction ID. List <EditLogInputStream> acc = new List <EditLogInputStream>(); EditLogInputStream elis; while ((elis = allStreams.Poll()) != null) { if (acc.IsEmpty()) { acc.AddItem(elis); } else { EditLogInputStream accFirst = acc[0]; long accFirstTxId = accFirst.GetFirstTxId(); if (accFirstTxId == elis.GetFirstTxId()) { // if we have a finalized log segment available at this txid, // we should throw out all in-progress segments at this txid if (elis.IsInProgress()) { if (accFirst.IsInProgress()) { acc.AddItem(elis); } } else { if (accFirst.IsInProgress()) { acc.Clear(); } acc.AddItem(elis); } } else { if (accFirstTxId < elis.GetFirstTxId()) { // try to read from the local logs first since the throughput should // be higher acc.Sort(LocalLogPreferenceComparator); outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId)); acc.Clear(); acc.AddItem(elis); } else { if (accFirstTxId > elis.GetFirstTxId()) { throw new RuntimeException("sorted set invariants violated! " + "Got stream with first txid " + elis.GetFirstTxId() + ", but the last firstTxId was " + accFirstTxId); } } } } } if (!acc.IsEmpty()) { acc.Sort(LocalLogPreferenceComparator); outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId)); acc.Clear(); } }
/// <exception cref="System.IO.IOException"/> private bool TryConvergeJournalSpool() { Preconditions.CheckState(bnState == BackupImage.BNState.JournalOnly, "bad state: %s" , bnState); // This section is unsynchronized so we can continue to apply // ahead of where we're reading, concurrently. Since the state // is JOURNAL_ONLY at this point, we know that lastAppliedTxId // doesn't change, and curSegmentTxId only increases while (lastAppliedTxId < editLog.GetCurSegmentTxId() - 1) { long target = editLog.GetCurSegmentTxId(); Log.Info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId + " to " + target); FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (); storage.InspectStorageDirs(inspector); editLog.RecoverUnclosedStreams(); IEnumerable <EditLogInputStream> editStreamsAll = editLog.SelectInputStreams(lastAppliedTxId , target - 1); // remove inprogress IList <EditLogInputStream> editStreams = Lists.NewArrayList(); foreach (EditLogInputStream s in editStreamsAll) { if (s.GetFirstTxId() != editLog.GetCurSegmentTxId()) { editStreams.AddItem(s); } } LoadEdits(editStreams, GetNamesystem()); } // now, need to load the in-progress file lock (this) { if (lastAppliedTxId != editLog.GetCurSegmentTxId() - 1) { Log.Debug("Logs rolled while catching up to current segment"); return(false); } // drop lock and try again to load local logs EditLogInputStream stream = null; ICollection <EditLogInputStream> editStreams = GetEditLog().SelectInputStreams(GetEditLog ().GetCurSegmentTxId(), GetEditLog().GetCurSegmentTxId()); foreach (EditLogInputStream s in editStreams) { if (s.GetFirstTxId() == GetEditLog().GetCurSegmentTxId()) { stream = s; } break; } if (stream == null) { Log.Warn("Unable to find stream starting with " + editLog.GetCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage" ); return(false); } try { long remainingTxns = GetEditLog().GetLastWrittenTxId() - lastAppliedTxId; Log.Info("Going to finish converging with remaining " + remainingTxns + " txns from in-progress stream " + stream); FSEditLogLoader loader = new FSEditLogLoader(GetNamesystem(), lastAppliedTxId); loader.LoadFSEdits(stream, lastAppliedTxId + 1); lastAppliedTxId = loader.GetLastAppliedTxId(); System.Diagnostics.Debug.Assert(lastAppliedTxId == GetEditLog().GetLastWrittenTxId ()); } finally { FSEditLog.CloseAllStreams(editStreams); } Log.Info("Successfully synced BackupNode with NameNode at txnid " + lastAppliedTxId ); SetState(BackupImage.BNState.InSync); } return(true); }
/// <exception cref="System.IO.IOException"/> public static long CountTransactionsInStream(EditLogInputStream @in) { FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.ValidateEditLog(@in ); return((validation.GetEndTxId() - @in.GetFirstTxId()) + 1); }