Ejemplo n.º 1
0
        public virtual void TestSimpleRead()
        {
            NamespaceInfo            nsi  = NewNSInfo();
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
                                                                             ("/hdfsjournal-simpleread"), nsi);

            bkjm.Format(nsi);
            long numTransactions     = 10000;
            EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
                                                            );

            for (long i = 1; i <= numTransactions; i++)
            {
                FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
                op.SetTransactionId(i);
                @out.Write(op);
            }
            @out.Close();
            bkjm.FinalizeLogSegment(1, numTransactions);
            IList <EditLogInputStream> @in = new AList <EditLogInputStream>();

            bkjm.SelectInputStreams(@in, 1, true);
            try
            {
                NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
                                                    (@in[0]));
            }
            finally
            {
                @in[0].Close();
            }
        }
        /// <summary>Test speculative read feature supported by bookkeeper.</summary>
        /// <remarks>
        /// Test speculative read feature supported by bookkeeper. Keep one bookie
        /// alive and sleep all the other bookies. Non spec client will hang for long
        /// time to read the entries from the bookkeeper.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestSpeculativeRead()
        {
            // starting 9 more servers
            for (int i = 1; i < 10; i++)
            {
                bks.AddItem(bkutil.NewBookie());
            }
            NamespaceInfo nsi          = NewNSInfo();
            Configuration conf         = new Configuration();
            int           ensembleSize = numLocalBookies + 9;

            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100);
            // sets 60 minute
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600);
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
                                                                             ("/hdfsjournal-specread"), nsi);

            bkjm.Format(nsi);
            long numTransactions     = 1000;
            EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
                                                            );

            for (long i_1 = 1; i_1 <= numTransactions; i_1++)
            {
                FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
                op.SetTransactionId(i_1);
                @out.Write(op);
            }
            @out.Close();
            bkjm.FinalizeLogSegment(1, numTransactions);
            IList <EditLogInputStream> @in = new AList <EditLogInputStream>();

            bkjm.SelectInputStreams(@in, 1, true);
            // sleep 9 bk servers. Now only one server is running and responding to the
            // clients
            CountDownLatch sleepLatch = new CountDownLatch(1);

            foreach (BookieServer bookie in bks)
            {
                SleepBookie(sleepLatch, bookie);
            }
            try
            {
                NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
                                                    (@in[0]));
            }
            finally
            {
                @in[0].Close();
                sleepLatch.CountDown();
                bkjm.Close();
            }
        }
Ejemplo n.º 3
0
        // expected
        /// <summary>Test ack quorum feature supported by bookkeeper.</summary>
        /// <remarks>
        /// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
        /// alive and sleep all the other bookies. Now the client would wait for the
        /// acknowledgement from the ack size bookies and after receiving the success
        /// response will continue writing. Non ack client will hang long time to add
        /// entries.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestAckQuorum()
        {
            // slow bookie
            newBookie = bkutil.NewBookie();
            // make quorum size and ensemble size same to avoid the interleave writing
            // of the ledger entries
            int ensembleSize = numBookies + 1;
            int quorumSize   = numBookies + 1;
            int ackSize      = numBookies;
            // ensure that the journal manager has to use all bookies,
            // so that a failure will fail the journal manager
            Configuration conf = new Configuration();

            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize);
            // sets 60 minutes
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600);
            NamespaceInfo            nsi  = NewNSInfo();
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
                                                                             ("/hdfsjournal-onebookiefailure"), nsi);

            bkjm.Format(nsi);
            CountDownLatch sleepLatch = new CountDownLatch(1);

            SleepBookie(sleepLatch, newBookie);
            EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
                                                            );
            int numTransactions = 100;

            for (long i = 1; i <= numTransactions; i++)
            {
                FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
                op.SetTransactionId(i);
                @out.Write(op);
            }
            @out.Close();
            bkjm.FinalizeLogSegment(1, numTransactions);
            IList <EditLogInputStream> @in = new AList <EditLogInputStream>();

            bkjm.SelectInputStreams(@in, 1, true);
            try
            {
                NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
                                                    (@in[0]));
            }
            finally
            {
                sleepLatch.CountDown();
                @in[0].Close();
                bkjm.Close();
            }
        }
Ejemplo n.º 4
0
        /// <exception cref="System.Exception"/>
        public virtual void TestDefaultAckQuorum()
        {
            newBookie = bkutil.NewBookie();
            int ensembleSize = numBookies + 1;
            int quorumSize   = numBookies + 1;
            // ensure that the journal manager has to use all bookies,
            // so that a failure will fail the journal manager
            Configuration conf = new Configuration();

            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
            // sets 2 secs
            conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 2);
            NamespaceInfo            nsi  = NewNSInfo();
            BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
                                                                             ("/hdfsjournal-onebookiefailure"), nsi);

            bkjm.Format(nsi);
            CountDownLatch sleepLatch = new CountDownLatch(1);

            SleepBookie(sleepLatch, newBookie);
            EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
                                                            );
            int numTransactions = 100;

            for (long i = 1; i <= numTransactions; i++)
            {
                FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
                op.SetTransactionId(i);
                @out.Write(op);
            }
            try
            {
                @out.Close();
                bkjm.FinalizeLogSegment(1, numTransactions);
                IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
                bkjm.SelectInputStreams(@in, 1, true);
                try
                {
                    NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
                                                        (@in[0]));
                }
                finally
                {
                    @in[0].Close();
                }
                NUnit.Framework.Assert.Fail("Should throw exception as not enough non-faulty bookies available!"
                                            );
            }
            catch (IOException)
            {
            }
        }