internal RedundantEditLogInputStream(ICollection <EditLogInputStream> streams, long
startTxId)
{
this.curIdx = 0;
this.prevTxId = (startTxId == HdfsConstants.InvalidTxid) ? HdfsConstants.InvalidTxid
: (startTxId - 1);
this.state = (streams.IsEmpty()) ? RedundantEditLogInputStream.State.Eof : RedundantEditLogInputStream.State
.SkipUntil;
this.prevException = null;
// EditLogInputStreams in a RedundantEditLogInputStream must be finalized,
// and can't be pre-transactional.
EditLogInputStream first = null;
foreach (EditLogInputStream s in streams)
{
Preconditions.CheckArgument(s.GetFirstTxId() != HdfsConstants.InvalidTxid, "invalid first txid in stream: %s"
, s);
Preconditions.CheckArgument(s.GetLastTxId() != HdfsConstants.InvalidTxid, "invalid last txid in stream: %s"
, s);
if (first == null)
{
first = s;
}
else
{
Preconditions.CheckArgument(s.GetFirstTxId() == first.GetFirstTxId(), "All streams in the RedundantEditLogInputStream must have the same "
+ "start transaction ID! " + first + " had start txId " + first.GetFirstTxId()
+ ", but " + s + " had start txId " + s.GetFirstTxId());
}
}
this.streams = Sharpen.Collections.ToArray(streams, new EditLogInputStream[0]);
// We sort the streams here so that the streams that end later come first.
Arrays.Sort(this.streams, new _IComparer_117());
}
/// <exception cref="System.IO.IOException"/>
private static EditLogInputStream GetJournalInputStream(JournalManager jm, long txId
, bool inProgressOk)
{
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
jm.SelectInputStreams(allStreams, txId, inProgressOk);
EditLogInputStream elis = null;
EditLogInputStream ret;
try
{
while ((elis = allStreams.Poll()) != null)
{
if (elis.GetFirstTxId() > txId)
{
break;
}
if (elis.GetLastTxId() < txId)
{
elis.Close();
continue;
}
elis.SkipUntil(txId);
ret = elis;
elis = null;
return(ret);
}
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream
[0]));
IOUtils.Cleanup(Log, elis);
}
return(null);
}
public static void ChainAndMakeRedundantStreams(ICollection <EditLogInputStream> outStreams
, PriorityQueue <EditLogInputStream> allStreams, long fromTxId)
{
// We want to group together all the streams that start on the same start
// transaction ID. To do this, we maintain an accumulator (acc) of all
// the streams we've seen at a given start transaction ID. When we see a
// higher start transaction ID, we select a stream from the accumulator and
// clear it. Then we begin accumulating streams with the new, higher start
// transaction ID.
List <EditLogInputStream> acc = new List <EditLogInputStream>();
EditLogInputStream elis;
while ((elis = allStreams.Poll()) != null)
{
if (acc.IsEmpty())
{
acc.AddItem(elis);
}
else
{
EditLogInputStream accFirst = acc[0];
long accFirstTxId = accFirst.GetFirstTxId();
if (accFirstTxId == elis.GetFirstTxId())
{
// if we have a finalized log segment available at this txid,
// we should throw out all in-progress segments at this txid
if (elis.IsInProgress())
{
if (accFirst.IsInProgress())
{
acc.AddItem(elis);
}
}
else
{
if (accFirst.IsInProgress())
{
acc.Clear();
}
acc.AddItem(elis);
}
}
else
{
if (accFirstTxId < elis.GetFirstTxId())
{
// try to read from the local logs first since the throughput should
// be higher
acc.Sort(LocalLogPreferenceComparator);
outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId));
acc.Clear();
acc.AddItem(elis);
}
else
{
if (accFirstTxId > elis.GetFirstTxId())
{
throw new RuntimeException("sorted set invariants violated! " + "Got stream with first txid "
+ elis.GetFirstTxId() + ", but the last firstTxId was " + accFirstTxId);
}
}
}
}
}
if (!acc.IsEmpty())
{
acc.Sort(LocalLogPreferenceComparator);
outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId));
acc.Clear();
}
}
/// <exception cref="System.IO.IOException"/>
public static long CountTransactionsInStream(EditLogInputStream @in)
{
FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.ValidateEditLog(@in
);
return((validation.GetEndTxId() - @in.GetFirstTxId()) + 1);
}