public virtual void TestEditLogFileNotExistsWhenReadingMetadata() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-editlogfile"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); try { // start new inprogress log segment with txid=1 // and write transactions till txid=50 string zkpath1 = StartAndFinalizeLogSegment(bkjm, 1, 50); // start new inprogress log segment with txid=51 // and write transactions till txid=100 string zkpath2 = StartAndFinalizeLogSegment(bkjm, 51, 100); // read the metadata from ZK. Here simulating the situation // when reading,the edit log metadata can be removed by purger thread. ZooKeeper zkspy = Org.Mockito.Mockito.Spy(BKJMUtil.ConnectZooKeeper()); bkjm.SetZooKeeper(zkspy); Org.Mockito.Mockito.DoThrow(new KeeperException.NoNodeException(zkpath2 + " doesn't exists" )).When(zkspy).GetData(zkpath2, false, null); IList <EditLogLedgerMetadata> ledgerList = bkjm.GetLedgerList(false); NUnit.Framework.Assert.AreEqual("List contains the metadata of non exists path.", 1, ledgerList.Count); NUnit.Framework.Assert.AreEqual("LogLedgerMetadata contains wrong zk paths.", zkpath1 , ledgerList[0].GetZkPath()); } finally { bkjm.Close(); } }
public virtual void TestEmptyInprogressLedger() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-emptyInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); for (long i_1 = 1; i_1 <= 100; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i_1); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(101, 200); bkjm.Close(); }
/// <summary>Test speculative read feature supported by bookkeeper.</summary> /// <remarks> /// Test speculative read feature supported by bookkeeper. Keep one bookie /// alive and sleep all the other bookies. Non spec client will hang for long /// time to read the entries from the bookkeeper. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestSpeculativeRead() { // starting 9 more servers for (int i = 1; i < 10; i++) { bks.AddItem(bkutil.NewBookie()); } NamespaceInfo nsi = NewNSInfo(); Configuration conf = new Configuration(); int ensembleSize = numLocalBookies + 9; conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100); // sets 60 minute conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-specread"), nsi); bkjm.Format(nsi); long numTransactions = 1000; EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i_1 = 1; i_1 <= numTransactions; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i_1); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); // sleep 9 bk servers. Now only one server is running and responding to the // clients CountDownLatch sleepLatch = new CountDownLatch(1); foreach (BookieServer bookie in bks) { SleepBookie(sleepLatch, bookie); } try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { @in[0].Close(); sleepLatch.CountDown(); bkjm.Close(); } }
public virtual void Teardown() { if (null != zkc) { zkc.Close(); } if (null != bkjm) { bkjm.Close(); } }
// expected /// <summary>Test ack quorum feature supported by bookkeeper.</summary> /// <remarks> /// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie /// alive and sleep all the other bookies. Now the client would wait for the /// acknowledgement from the ack size bookies and after receiving the success /// response will continue writing. Non ack client will hang long time to add /// entries. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestAckQuorum() { // slow bookie newBookie = bkutil.NewBookie(); // make quorum size and ensemble size same to avoid the interleave writing // of the ledger entries int ensembleSize = numBookies + 1; int quorumSize = numBookies + 1; int ackSize = numBookies; // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize); // sets 60 minutes conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-onebookiefailure"), nsi); bkjm.Format(nsi); CountDownLatch sleepLatch = new CountDownLatch(1); SleepBookie(sleepLatch, newBookie); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); int numTransactions = 100; for (long i = 1; i <= numTransactions; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { sleepLatch.CountDown(); @in[0].Close(); bkjm.Close(); } }
public virtual void TestConcurrentFormat() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-concurrentformat"); NamespaceInfo nsi = NewNSInfo(); // populate with data first BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); for (int i = 1; i < 100 * 2; i += 2) { bkjm.StartLogSegment(i, NameNodeLayoutVersion.CurrentLayoutVersion); bkjm.FinalizeLogSegment(i, i + 1); } bkjm.Close(); int numThreads = 40; IList <Callable <TestBookKeeperJournalManager.ThreadStatus> > threads = new AList <Callable <TestBookKeeperJournalManager.ThreadStatus> >(); CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i_1 = 0; i_1 < numThreads; i_1++) { threads.AddItem(new _Callable_784(uri, nsi, barrier)); } ExecutorService service = Executors.NewFixedThreadPool(numThreads); IList <Future <TestBookKeeperJournalManager.ThreadStatus> > statuses = service.InvokeAll (threads, 60, TimeUnit.Seconds); int numCompleted = 0; foreach (Future <TestBookKeeperJournalManager.ThreadStatus> s in statuses) { NUnit.Framework.Assert.IsTrue(s.IsDone()); NUnit.Framework.Assert.IsTrue("Thread threw invalid exception", s.Get() == TestBookKeeperJournalManager.ThreadStatus .Completed || s.Get() == TestBookKeeperJournalManager.ThreadStatus.Goodexception ); if (s.Get() == TestBookKeeperJournalManager.ThreadStatus.Completed) { numCompleted++; } } Log.Info("Completed " + numCompleted + " formats"); NUnit.Framework.Assert.IsTrue("No thread managed to complete formatting", numCompleted > 0); }
public virtual void TestCorruptInprogressNode() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(101); zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"), -1); bkjm = new BookKeeperJournalManager(conf, uri, nsi); try { bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating" + " an empty inprogess znode"); } catch (IOException e) { // correct behaviour NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains ("has no field named")); } finally { bkjm.Close(); } }
public TestBookKeeperJournalManager.ThreadStatus Call() { BookKeeperJournalManager bkjm = null; try { bkjm = new BookKeeperJournalManager(TestBookKeeperJournalManager.conf, uri, nsi); barrier.Await(); bkjm.Format(nsi); return(TestBookKeeperJournalManager.ThreadStatus.Completed); } catch (IOException ioe) { TestBookKeeperJournalManager.Log.Info("Exception formatting ", ioe); return(TestBookKeeperJournalManager.ThreadStatus.Goodexception); } catch (Exception ie) { TestBookKeeperJournalManager.Log.Error("Interrupted. Something is broken", ie); Sharpen.Thread.CurrentThread().Interrupt(); return(TestBookKeeperJournalManager.ThreadStatus.Badexception); } catch (Exception e) { TestBookKeeperJournalManager.Log.Error("Some other bad exception", e); return(TestBookKeeperJournalManager.ThreadStatus.Badexception); } finally { if (bkjm != null) { try { bkjm.Close(); } catch (IOException ioe) { TestBookKeeperJournalManager.Log.Error("Error closing journal manager", ioe); } } } }
public virtual void TestRefinalizeAlreadyFinalizedInprogress() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(1); string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100); NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode , null)); byte[] inprogressData = zkc.GetData(inprogressZNode, false, null); // finalize bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode , null)); zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode .Persistent); // should work fine bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); }