public virtual void TestEditLogFileNotExistsWhenReadingMetadata() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-editlogfile"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); try { // start new inprogress log segment with txid=1 // and write transactions till txid=50 string zkpath1 = StartAndFinalizeLogSegment(bkjm, 1, 50); // start new inprogress log segment with txid=51 // and write transactions till txid=100 string zkpath2 = StartAndFinalizeLogSegment(bkjm, 51, 100); // read the metadata from ZK. Here simulating the situation // when reading,the edit log metadata can be removed by purger thread. ZooKeeper zkspy = Org.Mockito.Mockito.Spy(BKJMUtil.ConnectZooKeeper()); bkjm.SetZooKeeper(zkspy); Org.Mockito.Mockito.DoThrow(new KeeperException.NoNodeException(zkpath2 + " doesn't exists" )).When(zkspy).GetData(zkpath2, false, null); IList <EditLogLedgerMetadata> ledgerList = bkjm.GetLedgerList(false); NUnit.Framework.Assert.AreEqual("List contains the metadata of non exists path.", 1, ledgerList.Count); NUnit.Framework.Assert.AreEqual("LogLedgerMetadata contains wrong zk paths.", zkpath1 , ledgerList[0].GetZkPath()); } finally { bkjm.Close(); } }
public virtual void TestTwoWriters() { long start = 1; NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-dualWriter"), nsi); bkjm1.Format(nsi); BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-dualWriter"), nsi); EditLogOutputStream out1 = bkjm1.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion ); try { bkjm2.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion); NUnit.Framework.Assert.Fail("Shouldn't have been able to open the second writer"); } catch (IOException ioe) { Log.Info("Caught exception as expected", ioe); } finally { out1.Close(); } }
public virtual void TestEmptyInprogressLedger() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-emptyInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); for (long i_1 = 1; i_1 <= 100; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i_1); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(101, 200); bkjm.Close(); }
public virtual void TestFailoverWithBK() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailover" ).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).ManageNameDfsSharedDirs(false).Build(); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); cluster.WaitActive(); cluster.TransitionToActive(0); Path p = new Path("/testBKJMfailover"); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); fs.Mkdirs(p); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.IsTrue(fs.Exists(p)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestSimpleRecovery() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-simplerecovery"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); @out.Abort(); @out.Close(); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false)); bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false )); NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false)); }
public virtual void TestSimpleRead() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-simpleread"), nsi); bkjm.Format(nsi); long numTransactions = 10000; EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= numTransactions; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { @in[0].Close(); } }
/// <summary>Test speculative read feature supported by bookkeeper.</summary> /// <remarks> /// Test speculative read feature supported by bookkeeper. Keep one bookie /// alive and sleep all the other bookies. Non spec client will hang for long /// time to read the entries from the bookkeeper. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestSpeculativeRead() { // starting 9 more servers for (int i = 1; i < 10; i++) { bks.AddItem(bkutil.NewBookie()); } NamespaceInfo nsi = NewNSInfo(); Configuration conf = new Configuration(); int ensembleSize = numLocalBookies + 9; conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100); // sets 60 minute conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-specread"), nsi); bkjm.Format(nsi); long numTransactions = 1000; EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i_1 = 1; i_1 <= numTransactions; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i_1); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); // sleep 9 bk servers. Now only one server is running and responding to the // clients CountDownLatch sleepLatch = new CountDownLatch(1); foreach (BookieServer bookie in bks) { SleepBookie(sleepLatch, bookie); } try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { @in[0].Close(); sleepLatch.CountDown(); bkjm.Close(); } }
/// <exception cref="System.Exception"/> public virtual void TestDefaultAckQuorum() { newBookie = bkutil.NewBookie(); int ensembleSize = numBookies + 1; int quorumSize = numBookies + 1; // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize); // sets 2 secs conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 2); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-onebookiefailure"), nsi); bkjm.Format(nsi); CountDownLatch sleepLatch = new CountDownLatch(1); SleepBookie(sleepLatch, newBookie); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); int numTransactions = 100; for (long i = 1; i <= numTransactions; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } try { @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { @in[0].Close(); } NUnit.Framework.Assert.Fail("Should throw exception as not enough non-faulty bookies available!" ); } catch (IOException) { } }
// expected /// <summary>Test ack quorum feature supported by bookkeeper.</summary> /// <remarks> /// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie /// alive and sleep all the other bookies. Now the client would wait for the /// acknowledgement from the ack size bookies and after receiving the success /// response will continue writing. Non ack client will hang long time to add /// entries. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestAckQuorum() { // slow bookie newBookie = bkutil.NewBookie(); // make quorum size and ensemble size same to avoid the interleave writing // of the ledger entries int ensembleSize = numBookies + 1; int quorumSize = numBookies + 1; int ackSize = numBookies; // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize); // sets 60 minutes conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-onebookiefailure"), nsi); bkjm.Format(nsi); CountDownLatch sleepLatch = new CountDownLatch(1); SleepBookie(sleepLatch, newBookie); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); int numTransactions = 100; for (long i = 1; i <= numTransactions; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, numTransactions); IList <EditLogInputStream> @in = new AList <EditLogInputStream>(); bkjm.SelectInputStreams(@in, 1, true); try { NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream (@in[0])); } finally { sleepLatch.CountDown(); @in[0].Close(); bkjm.Close(); } }
public virtual void TestInitializeBKSharedEdits() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); HAUtil.SetAllowStandbyReads(conf, true); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); MiniDFSNNTopology topology = MiniDFSNNTopology.SimpleHATopology(); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build (); cluster.WaitActive(); // Shutdown and clear the current filebased shared dir. cluster.ShutdownNameNodes(); FilePath shareddir = new FilePath(cluster.GetSharedEditsDir(0, 1)); NUnit.Framework.Assert.IsTrue("Initial Shared edits dir not fully deleted", FileUtil .FullyDelete(shareddir)); // Check namenodes should not start without shared dir. AssertCanNotStartNamenode(cluster, 0); AssertCanNotStartNamenode(cluster, 1); // Configure bkjm as new shared edits dir in both namenodes Configuration nn1Conf = cluster.GetConfiguration(0); Configuration nn2Conf = cluster.GetConfiguration(1); nn1Conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI ("/initializeSharedEdits").ToString()); nn2Conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI ("/initializeSharedEdits").ToString()); BKJMUtil.AddJournalManagerDefinition(nn1Conf); BKJMUtil.AddJournalManagerDefinition(nn2Conf); // Initialize the BKJM shared edits. NUnit.Framework.Assert.IsFalse(NameNode.InitializeSharedEdits(nn1Conf)); // NameNode should be able to start and should be in sync with BKJM as // shared dir AssertCanStartHANameNodes(cluster, conf, "/testBKJMInitialize"); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public override void SetupCluster() { Configuration conf = SetupCommonConfig(); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/checkpointing" + journalCount++).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf ("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetHttpPort(10001)).AddNN(new MiniDFSNNTopology.NNConf("nn2").SetHttpPort(10002))); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(1).ManageNameDfsSharedDirs (false).Build(); cluster.WaitActive(); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); cluster.TransitionToActive(0); }
public virtual void TestConcurrentFormat() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-concurrentformat"); NamespaceInfo nsi = NewNSInfo(); // populate with data first BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); for (int i = 1; i < 100 * 2; i += 2) { bkjm.StartLogSegment(i, NameNodeLayoutVersion.CurrentLayoutVersion); bkjm.FinalizeLogSegment(i, i + 1); } bkjm.Close(); int numThreads = 40; IList <Callable <TestBookKeeperJournalManager.ThreadStatus> > threads = new AList <Callable <TestBookKeeperJournalManager.ThreadStatus> >(); CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i_1 = 0; i_1 < numThreads; i_1++) { threads.AddItem(new _Callable_784(uri, nsi, barrier)); } ExecutorService service = Executors.NewFixedThreadPool(numThreads); IList <Future <TestBookKeeperJournalManager.ThreadStatus> > statuses = service.InvokeAll (threads, 60, TimeUnit.Seconds); int numCompleted = 0; foreach (Future <TestBookKeeperJournalManager.ThreadStatus> s in statuses) { NUnit.Framework.Assert.IsTrue(s.IsDone()); NUnit.Framework.Assert.IsTrue("Thread threw invalid exception", s.Get() == TestBookKeeperJournalManager.ThreadStatus .Completed || s.Get() == TestBookKeeperJournalManager.ThreadStatus.Goodexception ); if (s.Get() == TestBookKeeperJournalManager.ThreadStatus.Completed) { numCompleted++; } } Log.Info("Completed " + numCompleted + " formats"); NUnit.Framework.Assert.IsTrue("No thread managed to complete formatting", numCompleted > 0); }
public virtual void TestNumberOfTransactionsWithGaps() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-gaps"), nsi); bkjm.Format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j = 1; j <= DefaultSegmentSize; j++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, txid - 1); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, txid - 1), false)); } zkc.Delete(bkjm.FinalizedLedgerZNode(DefaultSegmentSize + 1, DefaultSegmentSize * 2), -1); long numTrans = bkjm.GetNumberOfTransactions(1, true); NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans); try { numTrans = bkjm.GetNumberOfTransactions(DefaultSegmentSize + 1, true); NUnit.Framework.Assert.Fail("Should have thrown corruption exception by this point" ); } catch (JournalManager.CorruptionException) { } // if we get here, everything is going good numTrans = bkjm.GetNumberOfTransactions((DefaultSegmentSize * 2) + 1, true); NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans); }
public virtual void TestMultiplePrimariesStarted() { Path p1 = new Path("/testBKJMMultiplePrimary"); MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailoverMultiple" ).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).ManageNameDfsSharedDirs(false).CheckExitOnShutdown(false).Build (); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); fs.Mkdirs(p1); nn1.GetRpcServer().RollEditLog(); cluster.TransitionToActive(1); fs = cluster.GetFileSystem(0); // get the older active server. try { fs.Delete(p1, true); NUnit.Framework.Assert.Fail("Log update on older active should cause it to exit"); } catch (RemoteException re) { NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException")); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestCorruptInprogressNode() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); @out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion); @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(101); zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"), -1); bkjm = new BookKeeperJournalManager(conf, uri, nsi); try { bkjm.RecoverUnfinalizedSegments(); NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating" + " an empty inprogess znode"); } catch (IOException e) { // correct behaviour NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains ("has no field named")); } finally { bkjm.Close(); } }
public virtual void TestNumberOfTransactionsWithInprogressAtEnd() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-inprogressAtEnd"), nsi); bkjm.Format(nsi); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j = 1; j <= DefaultSegmentSize; j++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, (txid - 1)); NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid - 1)), false)); } long start_1 = txid; EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); out_1.Write(op); } out_1.SetReadyToFlush(); out_1.Flush(); out_1.Abort(); out_1.Close(); long numTrans = bkjm.GetNumberOfTransactions(1, true); NUnit.Framework.Assert.AreEqual((txid - 1), numTrans); }
public virtual void TestRefinalizeAlreadyFinalizedInprogress() { URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger"); NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.Close(); string inprogressZNode = bkjm.InprogressZNode(1); string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100); NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode , null)); byte[] inprogressData = zkc.GetData(inprogressZNode, false, null); // finalize bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode , null)); NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode , null)); zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode .Persistent); // should work fine bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.RecoverUnfinalizedSegments(); bkjm.Close(); }
public virtual void TestEmptyInputStream() { ZooKeeper zk = BKJMUtil.ConnectZooKeeper(); BookKeeper bkc = new BookKeeper(new ClientConfiguration(), zk); try { LedgerHandle lh = bkc.CreateLedger(BookKeeper.DigestType.Crc32, Sharpen.Runtime.GetBytesForString ("foobar")); lh.Close(); EditLogLedgerMetadata metadata = new EditLogLedgerMetadata("/foobar", HdfsConstants .NamenodeLayoutVersion, lh.GetId(), unchecked ((int)(0x1234))); try { new BookKeeperEditLogInputStream(lh, metadata, -1); NUnit.Framework.Assert.Fail("Shouldn't get this far, should have thrown"); } catch (IOException ioe) { NUnit.Framework.Assert.IsTrue(ioe.Message.Contains("Invalid first bk entry to read" )); } metadata = new EditLogLedgerMetadata("/foobar", HdfsConstants.NamenodeLayoutVersion , lh.GetId(), unchecked ((int)(0x1234))); try { new BookKeeperEditLogInputStream(lh, metadata, 0); NUnit.Framework.Assert.Fail("Shouldn't get this far, should have thrown"); } catch (IOException ioe) { NUnit.Framework.Assert.IsTrue(ioe.Message.Contains("Invalid first bk entry to read" )); } } finally { bkc.Close(); zk.Close(); } }
public virtual void TestNumberOfTransactions() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-txncount"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(i); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(1, 100); long numTrans = bkjm.GetNumberOfTransactions(1, true); NUnit.Framework.Assert.AreEqual(100, numTrans); }
public virtual void SetUp() { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointCheckPeriodKey, 1); conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 5); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/bootstrapStandby" ).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); conf.SetBoolean(DFSConfigKeys.DfsImageCompressKey, true); conf.Set(DFSConfigKeys.DfsImageCompressionCodecKey, typeof(TestStandbyCheckpoints.SlowCodec ).GetCanonicalName()); CompressionCodecFactory.SetCodecClasses(conf, ImmutableList.Of <Type>(typeof(TestStandbyCheckpoints.SlowCodec ))); MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf ("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetHttpPort(10001)).AddNN(new MiniDFSNNTopology.NNConf("nn2").SetHttpPort(10002))); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(1).ManageNameDfsSharedDirs (false).Build(); cluster.WaitActive(); }
public virtual void TestNameNodeMultipleSwitchesUsingBKJM() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/correctEditLogSelection" ).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).ManageNameDfsSharedDirs(false).Build(); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); cluster.WaitActive(); cluster.TransitionToActive(0); nn1.GetRpcServer().RollEditLog(); // Roll Edits from current Active. // Transition to standby current active gracefully. cluster.TransitionToStandby(0); // Make the other Active and Roll edits multiple times cluster.TransitionToActive(1); nn2.GetRpcServer().RollEditLog(); nn2.GetRpcServer().RollEditLog(); // Now One more failover. So NN1 should be able to failover successfully. cluster.TransitionToStandby(1); cluster.TransitionToActive(0); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestFailoverWithFailingBKCluster() { int ensembleSize = numBookies + 1; BookieServer newBookie = bkutil.NewBookie(); NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); BookieServer replacementBookie = null; MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailoverWithFail" ).ToString()); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); BKJMUtil.AddJournalManagerDefinition(conf); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).ManageNameDfsSharedDirs(false).CheckExitOnShutdown(false).Build (); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); cluster.WaitActive(); cluster.TransitionToActive(0); Path p1 = new Path("/testBKJMFailingBKCluster1"); Path p2 = new Path("/testBKJMFailingBKCluster2"); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); fs.Mkdirs(p1); newBookie.Shutdown(); // will take down shared storage NUnit.Framework.Assert.AreEqual("New bookie didn't stop", numBookies, bkutil.CheckBookiesUp (numBookies, 10)); try { fs.Mkdirs(p2); NUnit.Framework.Assert.Fail("mkdirs should result in the NN exiting"); } catch (RemoteException re) { NUnit.Framework.Assert.IsTrue(re.GetClassName().Contains("ExitException")); } cluster.ShutdownNameNode(0); try { cluster.TransitionToActive(1); NUnit.Framework.Assert.Fail("Shouldn't have been able to transition with bookies down" ); } catch (ExitUtil.ExitException ee) { NUnit.Framework.Assert.IsTrue("Should shutdown due to required journal failure", ee.Message.Contains("starting log segment 3 failed for required journal")); } replacementBookie = bkutil.NewBookie(); NUnit.Framework.Assert.AreEqual("Replacement bookie didn't start", ensembleSize, bkutil.CheckBookiesUp(ensembleSize, 10)); cluster.TransitionToActive(1); // should work fine now NUnit.Framework.Assert.IsTrue(fs.Exists(p1)); NUnit.Framework.Assert.IsFalse(fs.Exists(p2)); } finally { newBookie.Shutdown(); if (replacementBookie != null) { replacementBookie.Shutdown(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestWriteRestartFrom1() { NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-restartFrom1"), nsi); bkjm.Format(nsi); long txid = 1; long start = txid; EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long j = 1; j <= DefaultSegmentSize; j++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, (txid - 1)); txid = 1; try { @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion); NUnit.Framework.Assert.Fail("Shouldn't be able to start another journal from " + txid + " when one already exists"); } catch (Exception ioe) { Log.Info("Caught exception as expected", ioe); } // test border case txid = DefaultSegmentSize; try { @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion); NUnit.Framework.Assert.Fail("Shouldn't be able to start another journal from " + txid + " when one already exists"); } catch (IOException ioe) { Log.Info("Caught exception as expected", ioe); } // open journal continuing from before txid = DefaultSegmentSize + 1; start = txid; @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion); NUnit.Framework.Assert.IsNotNull(@out); for (long j_1 = 1; j_1 <= DefaultSegmentSize; j_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.Close(); bkjm.FinalizeLogSegment(start, (txid - 1)); // open journal arbitarily far in the future txid = DefaultSegmentSize * 4; @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion); NUnit.Framework.Assert.IsNotNull(@out); }
public virtual void Setup() { zkc = BKJMUtil.ConnectZooKeeper(); }
public virtual void TestOneBookieFailure() { newBookie = bkutil.NewBookie(); BookieServer replacementBookie = null; try { int ensembleSize = numBookies + 1; NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager Configuration conf = new Configuration(); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize); conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize); long txid = 1; NamespaceInfo nsi = NewNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI ("/hdfsjournal-onebookiefailure"), nsi); bkjm.Format(nsi); EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion ); for (long i = 1; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); replacementBookie = bkutil.NewBookie(); NUnit.Framework.Assert.AreEqual("replacement bookie didn't start", ensembleSize + 1, bkutil.CheckBookiesUp(ensembleSize + 1, 10)); newBookie.Shutdown(); NUnit.Framework.Assert.AreEqual("New bookie didn't die", ensembleSize, bkutil.CheckBookiesUp (ensembleSize, 10)); for (long i_1 = 1; i_1 <= 3; i_1++) { FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance(); op.SetTransactionId(txid++); @out.Write(op); } @out.SetReadyToFlush(); @out.Flush(); } catch (Exception e) { Log.Error("Exception in test", e); throw; } finally { if (replacementBookie != null) { replacementBookie.Shutdown(); } newBookie.Shutdown(); if (bkutil.CheckBookiesUp(numBookies, 30) != numBookies) { Log.Warn("Not all bookies from this test shut down, expect errors"); } } }
public static void StartBK() { journalCount = 0; bkutil = new BKJMUtil(numBookies); bkutil.Start(); }
public static void SetupBookkeeper() { bkutil = new BKJMUtil(numBookies); bkutil.Start(); }