public virtual void TestSimpleRecovery() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-simplerecovery"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); @out.Abort(); @out.Close(); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false)); bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false)); }
public virtual void TestNumberOfTransactionsWithInprogressAtEnd() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-inprogressAtEnd"), nsi); bkjm.Format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j = 1; j <= DefaultSegmentSize; j++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, (txid - 1)); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid - 1)), false)); } long start_1 = txid; EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); out_1.Write(op); } out_1.SetReadyToFlush(); out_1.Flush(); out_1.Abort(); out_1.Close(); long numTrans = bkjm.GetNumberOfTransactions(1, true); NUnit.Framework.Assert.AreEqual((txid - 1), numTrans); }
public virtual void TestMissFinalizeAndNextStart() { // Logger 0: miss finalize(1-3) and start(4) TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies [0]).FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(3L)); TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies [0]).StartLogSegment(Org.Mockito.Mockito.Eq(4L), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion .CurrentLayoutVersion)); // Logger 1: fail at txn id 4 FailLoggerAtTxn(spies[1], 4L); QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true); EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion ); try { QJMTestUtil.WriteTxns(stm, 4, 1); NUnit.Framework.Assert.Fail("Did not fail to write"); } catch (QuorumException qe) { // Should fail, because logger 1 had an injected fault and // logger 0 should detect writer out of sync GenericTestUtils.AssertExceptionContains("Writer out of sync", qe); } finally { stm.Abort(); qjm.Close(); } // State: // Logger 0: 1-3 in-progress (since it missed finalize) // Logger 1: 1-3 finalized // Logger 2: 1-3 finalized, 4 in-progress with one txn // Shut down logger 2 so it doesn't participate in recovery cluster.GetJournalNode(2).StopAndJoin(0); qjm = CreateSpyingQJM(); long recovered = QJMTestUtil.RecoverAndReturnLastTxn(qjm); NUnit.Framework.Assert.AreEqual(3L, recovered); }
public virtual void TestCrashAtBeginningOfSegment() { QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true); WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion ); try { WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); } finally { stm.Abort(); } // Make a new QJM qjm = CloseLater(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil .Jid), QJMTestUtil.FakeNsinfo)); qjm.RecoverUnfinalizedSegments(); CheckRecovery(cluster, 1, 3); QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true); }
/// <summary> /// Test the case where, at the beginning of a segment, transactions /// have been written to one JN but not others. /// </summary> /// <exception cref="System.Exception"/> public virtual void DoTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn) { int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3; int nodeMissingSegment = (nodeWithOneTxn + 2) % 3; QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true); WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); cluster.GetJournalNode(nodeMissingSegment).StopAndJoin(0); // Open segment on 2/3 nodes EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion ); try { WaitForAllPendingCalls(qjm.GetLoggerSetForTests()); // Write transactions to only 1/3 nodes FailLoggerAtTxn(spies[nodeWithEmptySegment], 4); try { QJMTestUtil.WriteTxns(stm, 4, 1); NUnit.Framework.Assert.Fail("Did not fail even though 2/3 failed"); } catch (QuorumException qe) { GenericTestUtils.AssertExceptionContains("mock failure", qe); } } finally { stm.Abort(); } // Bring back the down JN. cluster.RestartJournalNode(nodeMissingSegment); // Make a new QJM. At this point, the state is as follows: // A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty) // B: nodeWithOneTxn: 1-3 finalized, 4_inprogress (1 txn) // C: nodeMissingSegment: 1-3 finalized GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithEmptySegment, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName (4)); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithOneTxn, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName (4)); GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeMissingSegment, QJMTestUtil .Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3)); // Stop one of the nodes. Since we run this test three // times, rotating the roles of the nodes, we'll test // all the permutations. cluster.GetJournalNode(2).StopAndJoin(0); qjm = CreateSpyingQJM(); qjm.RecoverUnfinalizedSegments(); if (nodeWithOneTxn == 0 || nodeWithOneTxn == 1) { // If the node that had the transaction committed was one of the nodes // that responded during recovery, then we should have recovered txid // 4. CheckRecovery(cluster, 4, 4); QJMTestUtil.WriteSegment(cluster, qjm, 5, 3, true); } else { // Otherwise, we should have recovered only 1-3 and should be able to // start a segment at 4. CheckRecovery(cluster, 1, 3); QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true); } }