internal RedundantEditLogInputStream(ICollection <EditLogInputStream> streams, long
startTxId)
{
this.curIdx = 0;
this.prevTxId = (startTxId == HdfsConstants.InvalidTxid) ? HdfsConstants.InvalidTxid
: (startTxId - 1);
this.state = (streams.IsEmpty()) ? RedundantEditLogInputStream.State.Eof : RedundantEditLogInputStream.State
.SkipUntil;
this.prevException = null;
// EditLogInputStreams in a RedundantEditLogInputStream must be finalized,
// and can't be pre-transactional.
EditLogInputStream first = null;
foreach (EditLogInputStream s in streams)
{
Preconditions.CheckArgument(s.GetFirstTxId() != HdfsConstants.InvalidTxid, "invalid first txid in stream: %s"
, s);
Preconditions.CheckArgument(s.GetLastTxId() != HdfsConstants.InvalidTxid, "invalid last txid in stream: %s"
, s);
if (first == null)
{
first = s;
}
else
{
Preconditions.CheckArgument(s.GetFirstTxId() == first.GetFirstTxId(), "All streams in the RedundantEditLogInputStream must have the same "
+ "start transaction ID! " + first + " had start txId " + first.GetFirstTxId()
+ ", but " + s + " had start txId " + s.GetFirstTxId());
}
}
this.streams = Sharpen.Collections.ToArray(streams, new EditLogInputStream[0]);
// We sort the streams here so that the streams that end later come first.
Arrays.Sort(this.streams, new _IComparer_117());
}
public virtual void TestReadURL()
{
HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>();
Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream
();
Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode
();
Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When
(conn).GetHeaderField("Content-Length");
URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>();
Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito
.Any <Uri>(), Matchers.AnyBoolean());
Uri url = new Uri("http://localhost/fakeLog");
EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants
.InvalidTxid, HdfsConstants.InvalidTxid, false);
// Read the edit log and verify that we got all of the data.
EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes
(elis);
Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is(
1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1));
// Check that length header was picked up.
NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length());
elis.Close();
}
public virtual void TestExcludeInProgressStreams()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams");
// Don't close the edit log once the files have been set up.
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, false);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// If we exclude the in-progess stream, we should only have 100 tx.
NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false)
);
EditLogInputStream elis = GetJournalInputStream(jm, 90, false);
try
{
FSEditLogOp lastReadOp = null;
while ((lastReadOp = elis.ReadOp()) != null)
{
NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100);
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
/// <summary>
/// Find out how many transactions we can read from a
/// FileJournalManager, starting at a given transaction ID.
/// </summary>
/// <param name="jm">The journal manager</param>
/// <param name="fromTxId">Transaction ID to start at</param>
/// <param name="inProgressOk">Should we consider edit logs that are not finalized?</param>
/// <returns>The number of transactions</returns>
/// <exception cref="System.IO.IOException"/>
internal static long GetNumberOfTransactions(FileJournalManager jm, long fromTxId
, bool inProgressOk, bool abortOnGap)
{
long numTransactions = 0;
long txId = fromTxId;
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
jm.SelectInputStreams(allStreams, fromTxId, inProgressOk);
EditLogInputStream elis = null;
try
{
while ((elis = allStreams.Poll()) != null)
{
try
{
elis.SkipUntil(txId);
while (true)
{
FSEditLogOp op = elis.ReadOp();
if (op == null)
{
break;
}
if (abortOnGap && (op.GetTransactionId() != txId))
{
Log.Info("getNumberOfTransactions: detected gap at txId " + fromTxId);
return(numTransactions);
}
txId = op.GetTransactionId() + 1;
numTransactions++;
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream
[0]));
}
return(numTransactions);
}
/// <seealso cref="CountEditLogOpTypes(Sharpen.FilePath)"/>
/// <exception cref="System.IO.IOException"/>
public static EnumMap <FSEditLogOpCodes, Holder <int> > CountEditLogOpTypes(EditLogInputStream
elis)
{
EnumMap <FSEditLogOpCodes, Holder <int> > opCounts = new EnumMap <FSEditLogOpCodes, Holder
<int> >(typeof(FSEditLogOpCodes));
FSEditLogOp op;
while ((op = elis.ReadOp()) != null)
{
Holder <int> i = opCounts[op.opCode];
if (i == null)
{
i = new Holder <int>(0);
opCounts[op.opCode] = i;
}
i.held++;
}
return(opCounts);
}
public virtual void TestReadFromMiddleOfEditLog()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
EditLogInputStream elis = GetJournalInputStream(jm, 5, true);
try
{
FSEditLogOp op = elis.ReadOp();
NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5);
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
/// <exception cref="System.IO.IOException"/>
private static EditLogInputStream GetJournalInputStream(JournalManager jm, long txId
, bool inProgressOk)
{
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
jm.SelectInputStreams(allStreams, txId, inProgressOk);
EditLogInputStream elis = null;
EditLogInputStream ret;
try
{
while ((elis = allStreams.Poll()) != null)
{
if (elis.GetFirstTxId() > txId)
{
break;
}
if (elis.GetLastTxId() < txId)
{
elis.Close();
continue;
}
elis.SkipUntil(txId);
ret = elis;
elis = null;
return(ret);
}
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream
[0]));
IOUtils.Cleanup(Log, elis);
}
return(null);
}
public static void ChainAndMakeRedundantStreams(ICollection <EditLogInputStream> outStreams
, PriorityQueue <EditLogInputStream> allStreams, long fromTxId)
{
// We want to group together all the streams that start on the same start
// transaction ID. To do this, we maintain an accumulator (acc) of all
// the streams we've seen at a given start transaction ID. When we see a
// higher start transaction ID, we select a stream from the accumulator and
// clear it. Then we begin accumulating streams with the new, higher start
// transaction ID.
List <EditLogInputStream> acc = new List <EditLogInputStream>();
EditLogInputStream elis;
while ((elis = allStreams.Poll()) != null)
{
if (acc.IsEmpty())
{
acc.AddItem(elis);
}
else
{
EditLogInputStream accFirst = acc[0];
long accFirstTxId = accFirst.GetFirstTxId();
if (accFirstTxId == elis.GetFirstTxId())
{
// if we have a finalized log segment available at this txid,
// we should throw out all in-progress segments at this txid
if (elis.IsInProgress())
{
if (accFirst.IsInProgress())
{
acc.AddItem(elis);
}
}
else
{
if (accFirst.IsInProgress())
{
acc.Clear();
}
acc.AddItem(elis);
}
}
else
{
if (accFirstTxId < elis.GetFirstTxId())
{
// try to read from the local logs first since the throughput should
// be higher
acc.Sort(LocalLogPreferenceComparator);
outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId));
acc.Clear();
acc.AddItem(elis);
}
else
{
if (accFirstTxId > elis.GetFirstTxId())
{
throw new RuntimeException("sorted set invariants violated! " + "Got stream with first txid "
+ elis.GetFirstTxId() + ", but the last firstTxId was " + accFirstTxId);
}
}
}
}
}
if (!acc.IsEmpty())
{
acc.Sort(LocalLogPreferenceComparator);
outStreams.AddItem(new RedundantEditLogInputStream(acc, fromTxId));
acc.Clear();
}
}
/// <exception cref="System.IO.IOException"/>
private bool TryConvergeJournalSpool()
{
Preconditions.CheckState(bnState == BackupImage.BNState.JournalOnly, "bad state: %s"
, bnState);
// This section is unsynchronized so we can continue to apply
// ahead of where we're reading, concurrently. Since the state
// is JOURNAL_ONLY at this point, we know that lastAppliedTxId
// doesn't change, and curSegmentTxId only increases
while (lastAppliedTxId < editLog.GetCurSegmentTxId() - 1)
{
long target = editLog.GetCurSegmentTxId();
Log.Info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId
+ " to " + target);
FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
();
storage.InspectStorageDirs(inspector);
editLog.RecoverUnclosedStreams();
IEnumerable <EditLogInputStream> editStreamsAll = editLog.SelectInputStreams(lastAppliedTxId
, target - 1);
// remove inprogress
IList <EditLogInputStream> editStreams = Lists.NewArrayList();
foreach (EditLogInputStream s in editStreamsAll)
{
if (s.GetFirstTxId() != editLog.GetCurSegmentTxId())
{
editStreams.AddItem(s);
}
}
LoadEdits(editStreams, GetNamesystem());
}
// now, need to load the in-progress file
lock (this)
{
if (lastAppliedTxId != editLog.GetCurSegmentTxId() - 1)
{
Log.Debug("Logs rolled while catching up to current segment");
return(false);
}
// drop lock and try again to load local logs
EditLogInputStream stream = null;
ICollection <EditLogInputStream> editStreams = GetEditLog().SelectInputStreams(GetEditLog
().GetCurSegmentTxId(), GetEditLog().GetCurSegmentTxId());
foreach (EditLogInputStream s in editStreams)
{
if (s.GetFirstTxId() == GetEditLog().GetCurSegmentTxId())
{
stream = s;
}
break;
}
if (stream == null)
{
Log.Warn("Unable to find stream starting with " + editLog.GetCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage"
);
return(false);
}
try
{
long remainingTxns = GetEditLog().GetLastWrittenTxId() - lastAppliedTxId;
Log.Info("Going to finish converging with remaining " + remainingTxns + " txns from in-progress stream "
+ stream);
FSEditLogLoader loader = new FSEditLogLoader(GetNamesystem(), lastAppliedTxId);
loader.LoadFSEdits(stream, lastAppliedTxId + 1);
lastAppliedTxId = loader.GetLastAppliedTxId();
System.Diagnostics.Debug.Assert(lastAppliedTxId == GetEditLog().GetLastWrittenTxId
());
}
finally
{
FSEditLog.CloseAllStreams(editStreams);
}
Log.Info("Successfully synced BackupNode with NameNode at txnid " + lastAppliedTxId
);
SetState(BackupImage.BNState.InSync);
}
return(true);
}
/// <exception cref="System.IO.IOException"/>
public static long CountTransactionsInStream(EditLogInputStream @in)
{
FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.ValidateEditLog(@in
);
return((validation.GetEndTxId() - @in.GetFirstTxId()) + 1);
}
