internal RedundantEditLogInputStream(ICollection <EditLogInputStream> streams, long startTxId) { this.curIdx = 0; this.prevTxId = (startTxId == HdfsConstants.InvalidTxid) ? HdfsConstants.InvalidTxid : (startTxId - 1); this.state = (streams.IsEmpty()) ? RedundantEditLogInputStream.State.Eof : RedundantEditLogInputStream.State .SkipUntil; this.prevException = null; // EditLogInputStreams in a RedundantEditLogInputStream must be finalized, // and can't be pre-transactional. EditLogInputStream first = null; foreach (EditLogInputStream s in streams) { Preconditions.CheckArgument(s.GetFirstTxId() != HdfsConstants.InvalidTxid, "invalid first txid in stream: %s" , s); Preconditions.CheckArgument(s.GetLastTxId() != HdfsConstants.InvalidTxid, "invalid last txid in stream: %s" , s); if (first == null) { first = s; } else { Preconditions.CheckArgument(s.GetFirstTxId() == first.GetFirstTxId(), "All streams in the RedundantEditLogInputStream must have the same " + "start transaction ID! " + first + " had start txId " + first.GetFirstTxId() + ", but " + s + " had start txId " + s.GetFirstTxId()); } } this.streams = Sharpen.Collections.ToArray(streams, new EditLogInputStream[0]); // We sort the streams here so that the streams that end later come first. Arrays.Sort(this.streams, new _IComparer_117()); }
/// <exception cref="System.IO.IOException"/> private static EditLogInputStream GetJournalInputStream(JournalManager jm, long txId , bool inProgressOk) { PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream >(64, JournalSet.EditLogInputStreamComparator); jm.SelectInputStreams(allStreams, txId, inProgressOk); EditLogInputStream elis = null; EditLogInputStream ret; try { while ((elis = allStreams.Poll()) != null) { if (elis.GetFirstTxId() > txId) { break; } if (elis.GetLastTxId() < txId) { elis.Close(); continue; } elis.SkipUntil(txId); ret = elis; elis = null; return(ret); } } finally { IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream [0])); IOUtils.Cleanup(Log, elis); } return(null); }
public static void ChainAndMakeRedundantStreams(ICollection <EditLogInputStream> outStreams , PriorityQueue <EditLogInputStream> allStreams, long fromTxId) { // We want to group together all the streams that start on the same start // transaction ID. To do this, we maintain an accumulator (acc) of all // the streams we've seen at a given start transaction ID. When we see a // higher start transaction ID, we select a stream from the accumulator and // clear it. Then we begin accumulating streams with the new, higher start // transaction ID. List <EditLogInputStream> acc = new List <EditLogInputStream>(); EditLogInputStream elis; while ((elis = allStreams.Poll()) != null) { if (acc.IsEmpty()) { acc.AddItem(elis); } else { EditLogInputStream accFirst = acc[0]; long accFirstTxId = accFirst.GetFirstTxId(); if (accFirstTxId == elis.GetFirstTxId()) { // if we have a finalized log segment available at this txid, // we should throw out all in-progress segments at this txid if (elis.IsInProgress()) { if (accFirst.IsInProgress()) { acc.AddItem(elis); } } else { if (accFirst.IsInProgress()) { acc.Clear(); } acc.AddItem(elis); } } else { if (accFirstTxId < elis.GetFirstTxId()) { // try to read from the local logs first since the throughput should // be higher acc.Sort(LocalLogPreferenceComparator); outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId)); acc.Clear(); acc.AddItem(elis); } else { if (accFirstTxId > elis.GetFirstTxId()) { throw new RuntimeException("sorted set invariants violated! " + "Got stream with first txid " + elis.GetFirstTxId() + ", but the last firstTxId was " + accFirstTxId); } } } } } if (!acc.IsEmpty()) { acc.Sort(LocalLogPreferenceComparator); outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId)); acc.Clear(); } }
/// <exception cref="System.IO.IOException"/> public static long CountTransactionsInStream(EditLogInputStream @in) { FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.ValidateEditLog(@in ); return((validation.GetEndTxId() - @in.GetFirstTxId()) + 1); }