public virtual void TestEmptyInprogressLedger() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-emptyInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); for (long i_1 = 1; i_1 <= 100; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i_1); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(101, 200); bkjm.Close(); }
public virtual void TestSimpleRecovery() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-simplerecovery"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); @out.Abort(); @out.Close(); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false)); bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false)); }
public virtual void TestCorruptInprogressNode() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(101); zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"), -1); bkjm = new BookKeeperJournalManager(conf, uri, nsi); try { bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating" + " an empty inprogess znode"); } catch (IOException e) { // correct behaviour NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains ("has no field named")); } finally { bkjm.Close(); } }
public virtual void TestRefinalizeAlreadyFinalizedInprogress() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(1); string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100); NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode , null)); byte[] inprogressData = zkc.GetData(inprogressZNode, false, null); // finalize bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode , null)); zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode .Persistent); // should work fine bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); }
public virtual void TestAllBookieFailure() { // bookie to fail newBookie = bkutil.NewBookie(); BookieServer replacementBookie = null; try { int ensembleSize = numBookies + 1; NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); long txid = 1; NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-allbookiefailure"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); newBookie.Shutdown(); NUnit.Framework.Assert.AreEqual("New bookie didn't die", numBookies, bkutil.CheckBookiesUp (numBookies, 10)); try { for (long i_1 = 1; i_1 <= 3; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); NUnit.Framework.Assert.Fail("should not get to this stage"); } catch (IOException ioe) { Log.Debug("Error writing to bookkeeper", ioe); NUnit.Framework.Assert.IsTrue("Invalid exception message", ioe.Message.Contains("Failed to write to bookkeeper" )); } replacementBookie = bkutil.NewBookie(); NUnit.Framework.Assert.AreEqual("New bookie didn't start", numBookies + 1, bkutil .CheckBookiesUp(numBookies + 1, 10)); bkjm.RecoverUnfinalizedSegments(); @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion); for (long i_2 = 1; i_2 <= 3; i_2++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); } catch (Exception e) { Log.Error("Exception in test", e); throw; } finally { if (replacementBookie != null) { replacementBookie.Shutdown(); } newBookie.Shutdown(); if (bkutil.CheckBookiesUp(numBookies, 30) != numBookies) { Log.Warn("Not all bookies from this test shut down, expect errors"); } } }