public virtual void TestSimpleRecovery()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-simplerecovery"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
@out.Abort();
@out.Close();
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false));
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false));
}
public virtual void TestNumberOfTransactionsWithInprogressAtEnd()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-inprogressAtEnd"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid
- 1)), false));
}
long start_1 = txid;
EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
out_1.Write(op);
}
out_1.SetReadyToFlush();
out_1.Flush();
out_1.Abort();
out_1.Close();
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual((txid - 1), numTrans);
}
public virtual void TestMissFinalizeAndNextStart()
{
// Logger 0: miss finalize(1-3) and start(4)
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(3L));
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).StartLogSegment(Org.Mockito.Mockito.Eq(4L), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion
.CurrentLayoutVersion));
// Logger 1: fail at txn id 4
FailLoggerAtTxn(spies[1], 4L);
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail to write");
}
catch (QuorumException qe)
{
// Should fail, because logger 1 had an injected fault and
// logger 0 should detect writer out of sync
GenericTestUtils.AssertExceptionContains("Writer out of sync", qe);
}
finally
{
stm.Abort();
qjm.Close();
}
// State:
// Logger 0: 1-3 in-progress (since it missed finalize)
// Logger 1: 1-3 finalized
// Logger 2: 1-3 finalized, 4 in-progress with one txn
// Shut down logger 2 so it doesn't participate in recovery
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
long recovered = QJMTestUtil.RecoverAndReturnLastTxn(qjm);
NUnit.Framework.Assert.AreEqual(3L, recovered);
}
public virtual void TestCrashAtBeginningOfSegment()
{
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
}
finally
{
stm.Abort();
}
// Make a new QJM
qjm = CloseLater(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil
.Jid), QJMTestUtil.FakeNsinfo));
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 3);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
}
/// <summary>
/// Test the case where, at the beginning of a segment, transactions
/// have been written to one JN but not others.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void DoTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
{
int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.GetJournalNode(nodeMissingSegment).StopAndJoin(0);
// Open segment on 2/3 nodes
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
// Write transactions to only 1/3 nodes
FailLoggerAtTxn(spies[nodeWithEmptySegment], 4);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail even though 2/3 failed");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
}
finally
{
stm.Abort();
}
// Bring back the down JN.
cluster.RestartJournalNode(nodeMissingSegment);
// Make a new QJM. At this point, the state is as follows:
// A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)
// B: nodeWithOneTxn: 1-3 finalized, 4_inprogress (1 txn)
// C: nodeMissingSegment: 1-3 finalized
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithEmptySegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithOneTxn, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeMissingSegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3));
// Stop one of the nodes. Since we run this test three
// times, rotating the roles of the nodes, we'll test
// all the permutations.
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
qjm.RecoverUnfinalizedSegments();
if (nodeWithOneTxn == 0 || nodeWithOneTxn == 1)
{
// If the node that had the transaction committed was one of the nodes
// that responded during recovery, then we should have recovered txid
// 4.
CheckRecovery(cluster, 4, 4);
QJMTestUtil.WriteSegment(cluster, qjm, 5, 3, true);
}
else
{
// Otherwise, we should have recovered only 1-3 and should be able to
// start a segment at 4.
CheckRecovery(cluster, 1, 3);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
}
}