/// <summary>Test finalizing a segment after some batch of edits were missed.</summary>
/// <remarks>
/// Test finalizing a segment after some batch of edits were missed.
/// This should fail, since we validate the log before finalization.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFinalizeWhenEditsAreMissed()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 3, QJMTestUtil.CreateTxnData(1, 3));
// Try to finalize up to txn 6, even though we only wrote up to txn 3.
try
{
journal.FinalizeLogSegment(MakeRI(3), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("but only written up to txid 3", e);
}
// Check that, even if we re-construct the journal by scanning the
// disk, we don't allow finalizing incorrectly.
journal.Close();
journal = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular
, mockErrorReporter);
try
{
journal.FinalizeLogSegment(MakeRI(4), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("disk only contains up to txid 3", e);
}
}
/// <summary>
/// Test that, if the writer crashes at the very beginning of a segment,
/// before any transactions are written, that the next newEpoch() call
/// returns the prior segment txid as its most recent segment.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestNewEpochAtBeginningOfSegment()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 2, QJMTestUtil.CreateTxnData(1, 2));
journal.FinalizeLogSegment(MakeRI(3), 1, 2);
journal.StartLogSegment(MakeRI(4), 3, NameNodeLayoutVersion.CurrentLayoutVersion);
QJournalProtocolProtos.NewEpochResponseProto resp = journal.NewEpoch(FakeNsinfo,
2);
NUnit.Framework.Assert.AreEqual(1, resp.GetLastSegmentTxId());
}
/// <exception cref="System.Exception"/>
public virtual void TestMaintainCommittedTxId()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
// Send txids 1-3, with a request indicating only 0 committed
journal.Journal(new RequestInfo(Jid, 1, 2, 0), 1, 1, 3, QJMTestUtil.CreateTxnData
(1, 3));
NUnit.Framework.Assert.AreEqual(0, journal.GetCommittedTxnIdForTests());
// Send 4-6, with request indicating that through 3 is committed.
journal.Journal(new RequestInfo(Jid, 1, 3, 3), 1, 4, 3, QJMTestUtil.CreateTxnData
(4, 6));
NUnit.Framework.Assert.AreEqual(3, journal.GetCommittedTxnIdForTests());
}
/// <summary>
/// Assume that a client is writing to a journal, but loses its connection
/// in the middle of a segment.
/// </summary>
/// <remarks>
/// Assume that a client is writing to a journal, but loses its connection
/// in the middle of a segment. Thus, any future journal() calls in that
/// segment may fail, because some txns were missed while the connection was
/// down.
/// Eventually, the connection comes back, and the NN tries to start a new
/// segment at a higher txid. This should abort the old one and succeed.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAbortOldSegmentIfFinalizeIsMissed()
{
journal.NewEpoch(FakeNsinfo, 1);
// Start a segment at txid 1, and write a batch of 3 txns.
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 3, QJMTestUtil.CreateTxnData(1, 3));
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(1));
// Try to start new segment at txid 6, this should abort old segment and
// then succeed, allowing us to write txid 6-9.
journal.StartLogSegment(MakeRI(3), 6, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(4), 6, 6, 3, QJMTestUtil.CreateTxnData(6, 3));
// The old segment should *not* be finalized.
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(1));
GenericTestUtils.AssertExists(journal.GetStorage().GetInProgressEditLog(6));
}
/// <exception cref="System.Exception"/>
public virtual void TestRestartJournal()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 2, QJMTestUtil.CreateTxnData(1, 2));
// Don't finalize.
string storageString = journal.GetStorage().ToColonSeparatedString();
System.Console.Error.WriteLine("storage string: " + storageString);
journal.Close();
// close to unlock the storage dir
// Now re-instantiate, make sure history is still there
journal = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular
, mockErrorReporter);
// The storage info should be read, even if no writer has taken over.
NUnit.Framework.Assert.AreEqual(storageString, journal.GetStorage().ToColonSeparatedString
());
NUnit.Framework.Assert.AreEqual(1, journal.GetLastPromisedEpoch());
QJournalProtocolProtos.NewEpochResponseProtoOrBuilder newEpoch = journal.NewEpoch
(FakeNsinfo, 2);
NUnit.Framework.Assert.AreEqual(1, newEpoch.GetLastSegmentTxId());
}
/// <exception cref="System.Exception"/>
public virtual void TestHttpServer()
{
string urlRoot = jn.GetHttpServerURI();
// Check default servlets.
string pageContents = DFSTestUtil.UrlGet(new Uri(urlRoot + "/jmx"));
NUnit.Framework.Assert.IsTrue("Bad contents: " + pageContents, pageContents.Contains
("Hadoop:service=JournalNode,name=JvmMetrics"));
// Create some edits on server side
byte[] EditsData = QJMTestUtil.CreateTxnData(1, 3);
IPCLoggerChannel ch = new IPCLoggerChannel(conf, FakeNsinfo, journalId, jn.GetBoundIpcAddress
());
ch.NewEpoch(1).Get();
ch.SetEpoch(1);
ch.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion).Get();
ch.SendEdits(1L, 1, 3, EditsData).Get();
ch.FinalizeLogSegment(1, 3).Get();
// Attempt to retrieve via HTTP, ensure we get the data back
// including the header we expected
byte[] retrievedViaHttp = DFSTestUtil.UrlGetBytes(new Uri(urlRoot + "/getJournal?segmentTxId=1&jid="
+ journalId));
byte[] expected = Bytes.Concat(Ints.ToByteArray(HdfsConstants.NamenodeLayoutVersion
), (new byte[] { 0, 0, 0, 0 }), EditsData);
// layout flags section
Assert.AssertArrayEquals(expected, retrievedViaHttp);
// Attempt to fetch a non-existent file, check that we get an
// error status code
Uri badUrl = new Uri(urlRoot + "/getJournal?segmentTxId=12345&jid=" + journalId);
HttpURLConnection connection = (HttpURLConnection)badUrl.OpenConnection();
try
{
NUnit.Framework.Assert.AreEqual(404, connection.GetResponseCode());
}
finally
{
connection.Disconnect();
}
}
/// <summary>
/// Test behavior of startLogSegment() when a segment with the
/// same transaction ID already exists.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestStartLogSegmentWhenAlreadyExists()
{
journal.NewEpoch(FakeNsinfo, 1);
// Start a segment at txid 1, and write just 1 transaction. This
// would normally be the START_LOG_SEGMENT transaction.
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 1, QJMTestUtil.CreateTxnData(1, 1));
// Try to start new segment at txid 1, this should succeed, because
// we are allowed to re-start a segment if we only ever had the
// START_LOG_SEGMENT transaction logged.
journal.StartLogSegment(MakeRI(3), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(4), 1, 1, 1, QJMTestUtil.CreateTxnData(1, 1));
// This time through, write more transactions afterwards, simulating
// real user transactions.
journal.Journal(MakeRI(5), 1, 2, 3, QJMTestUtil.CreateTxnData(2, 3));
try
{
journal.StartLogSegment(MakeRI(6), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.Fail("Did not fail to start log segment which would overwrite "
+ "an existing one");
}
catch (InvalidOperationException ise)
{
GenericTestUtils.AssertExceptionContains("seems to contain valid transactions", ise
);
}
journal.FinalizeLogSegment(MakeRI(7), 1, 4);
// Ensure that we cannot overwrite a finalized segment
try
{
journal.StartLogSegment(MakeRI(8), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.Fail("Did not fail to start log segment which would overwrite "
+ "an existing one");
}
catch (InvalidOperationException ise)
{
GenericTestUtils.AssertExceptionContains("have a finalized segment", ise);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestReturnsSegmentInfoAtEpochTransition()
{
ch.NewEpoch(1).Get();
ch.SetEpoch(1);
ch.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion).Get();
ch.SendEdits(1L, 1, 2, QJMTestUtil.CreateTxnData(1, 2)).Get();
// Switch to a new epoch without closing earlier segment
QJournalProtocolProtos.NewEpochResponseProto response = ch.NewEpoch(2).Get();
ch.SetEpoch(2);
NUnit.Framework.Assert.AreEqual(1, response.GetLastSegmentTxId());
ch.FinalizeLogSegment(1, 2).Get();
// Switch to a new epoch after just closing the earlier segment.
response = ch.NewEpoch(3).Get();
ch.SetEpoch(3);
NUnit.Framework.Assert.AreEqual(1, response.GetLastSegmentTxId());
// Start a segment but don't write anything, check newEpoch segment info
ch.StartLogSegment(3, NameNodeLayoutVersion.CurrentLayoutVersion).Get();
response = ch.NewEpoch(4).Get();
ch.SetEpoch(4);
// Because the new segment is empty, it is equivalent to not having
// started writing it. Hence, we should return the prior segment txid.
NUnit.Framework.Assert.AreEqual(1, response.GetLastSegmentTxId());
}
/// <summary>
/// Test that the JournalNode performs correctly as a Paxos
/// <em>Acceptor</em> process.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestAcceptRecoveryBehavior()
{
// We need to run newEpoch() first, or else we have no way to distinguish
// different proposals for the same decision.
try
{
ch.PrepareRecovery(1L).Get();
NUnit.Framework.Assert.Fail("Did not throw IllegalState when trying to run paxos without an epoch"
);
}
catch (ExecutionException ise)
{
GenericTestUtils.AssertExceptionContains("bad epoch", ise);
}
ch.NewEpoch(1).Get();
ch.SetEpoch(1);
// prepare() with no previously accepted value and no logs present
QJournalProtocolProtos.PrepareRecoveryResponseProto prep = ch.PrepareRecovery(1L)
.Get();
System.Console.Error.WriteLine("Prep: " + prep);
NUnit.Framework.Assert.IsFalse(prep.HasAcceptedInEpoch());
NUnit.Framework.Assert.IsFalse(prep.HasSegmentState());
// Make a log segment, and prepare again -- this time should see the
// segment existing.
ch.StartLogSegment(1L, NameNodeLayoutVersion.CurrentLayoutVersion).Get();
ch.SendEdits(1L, 1L, 1, QJMTestUtil.CreateTxnData(1, 1)).Get();
prep = ch.PrepareRecovery(1L).Get();
System.Console.Error.WriteLine("Prep: " + prep);
NUnit.Framework.Assert.IsFalse(prep.HasAcceptedInEpoch());
NUnit.Framework.Assert.IsTrue(prep.HasSegmentState());
// accept() should save the accepted value in persistent storage
ch.AcceptRecovery(prep.GetSegmentState(), new Uri("file:///dev/null")).Get();
// So another prepare() call from a new epoch would return this value
ch.NewEpoch(2);
ch.SetEpoch(2);
prep = ch.PrepareRecovery(1L).Get();
NUnit.Framework.Assert.AreEqual(1L, prep.GetAcceptedInEpoch());
NUnit.Framework.Assert.AreEqual(1L, prep.GetSegmentState().GetEndTxId());
// A prepare() or accept() call from an earlier epoch should now be rejected
ch.SetEpoch(1);
try
{
ch.PrepareRecovery(1L).Get();
NUnit.Framework.Assert.Fail("prepare from earlier epoch not rejected");
}
catch (ExecutionException ioe)
{
GenericTestUtils.AssertExceptionContains("epoch 1 is less than the last promised epoch 2"
, ioe);
}
try
{
ch.AcceptRecovery(prep.GetSegmentState(), new Uri("file:///dev/null")).Get();
NUnit.Framework.Assert.Fail("accept from earlier epoch not rejected");
}
catch (ExecutionException ioe)
{
GenericTestUtils.AssertExceptionContains("epoch 1 is less than the last promised epoch 2"
, ioe);
}
}
