public virtual void TestWriteEdits() { EditLogOutputStream stm = CreateLogSegment(); QJMTestUtil.WriteOp(stm, 1); QJMTestUtil.WriteOp(stm, 2); stm.SetReadyToFlush(); QJMTestUtil.WriteOp(stm, 3); // The flush should log txn 1-2 FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq (1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>()); FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq (1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>()); FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq (1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>()); stm.Flush(); // Another flush should now log txn #3 stm.SetReadyToFlush(); FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq (3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq (3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq (3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); stm.Flush(); }
public virtual void TestSimpleRecovery() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-simplerecovery"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); @out.Abort(); @out.Close(); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false)); bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false)); }
/// <exception cref="System.IO.IOException"/> public static void WriteTxns(EditLogOutputStream stm, long startTxId, int numTxns ) { for (long txid = startTxId; txid < startTxId + numTxns; txid++) { WriteOp(stm, txid); } stm.SetReadyToFlush(); stm.Flush(); }
public virtual void TestNumberOfTransactionsWithInprogressAtEnd() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-inprogressAtEnd"), nsi); bkjm.Format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j = 1; j <= DefaultSegmentSize; j++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, (txid - 1)); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid - 1)), false)); } long start_1 = txid; EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); out_1.Write(op); } out_1.SetReadyToFlush(); out_1.Flush(); out_1.Abort(); out_1.Close(); long numTrans = bkjm.GetNumberOfTransactions(1, true); NUnit.Framework.Assert.AreEqual((txid - 1), numTrans); }
public virtual void TestWriteEditsOneSlow() { EditLogOutputStream stm = CreateLogSegment(); QJMTestUtil.WriteOp(stm, 1); stm.SetReadyToFlush(); // Make the first two logs respond immediately FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq (1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq (1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); // And the third log not respond SettableFuture <Void> slowLog = SettableFuture.Create(); Org.Mockito.Mockito.DoReturn(slowLog).When(spyLoggers[2]).SendEdits(Matchers.AnyLong (), Matchers.Eq(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>()); stm.Flush(); Org.Mockito.Mockito.Verify(spyLoggers[0]).SetCommittedTxId(1L); }
public virtual void TestOneBookieFailure() { newBookie = bkutil.NewBookie(); BookieServer replacementBookie = null; try { int ensembleSize = numBookies + 1; NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); long txid = 1; NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-onebookiefailure"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); replacementBookie = bkutil.NewBookie(); NUnit.Framework.Assert.AreEqual("replacement bookie didn't start", ensembleSize + 1, bkutil.CheckBookiesUp(ensembleSize + 1, 10)); newBookie.Shutdown(); NUnit.Framework.Assert.AreEqual("New bookie didn't die", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); for (long i_1 = 1; i_1 <= 3; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); } catch (Exception e) { Log.Error("Exception in test", e); throw; } finally { if (replacementBookie != null) { replacementBookie.Shutdown(); } newBookie.Shutdown(); if (bkutil.CheckBookiesUp(numBookies, 30) != numBookies) { Log.Warn("Not all bookies from this test shut down, expect errors"); } } }