public virtual void TestEditLogFileNotExistsWhenReadingMetadata()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-editlogfile");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
try
{
// start new inprogress log segment with txid=1
// and write transactions till txid=50
string zkpath1 = StartAndFinalizeLogSegment(bkjm, 1, 50);
// start new inprogress log segment with txid=51
// and write transactions till txid=100
string zkpath2 = StartAndFinalizeLogSegment(bkjm, 51, 100);
// read the metadata from ZK. Here simulating the situation
// when reading,the edit log metadata can be removed by purger thread.
ZooKeeper zkspy = Org.Mockito.Mockito.Spy(BKJMUtil.ConnectZooKeeper());
bkjm.SetZooKeeper(zkspy);
Org.Mockito.Mockito.DoThrow(new KeeperException.NoNodeException(zkpath2 + " doesn't exists"
)).When(zkspy).GetData(zkpath2, false, null);
IList <EditLogLedgerMetadata> ledgerList = bkjm.GetLedgerList(false);
NUnit.Framework.Assert.AreEqual("List contains the metadata of non exists path.",
1, ledgerList.Count);
NUnit.Framework.Assert.AreEqual("LogLedgerMetadata contains wrong zk paths.", zkpath1
, ledgerList[0].GetZkPath());
}
finally
{
bkjm.Close();
}
}
public virtual void TestEmptyInprogressLedger()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-emptyInprogressLedger");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
@out.Close();
bkjm.Close();
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
for (long i_1 = 1; i_1 <= 100; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i_1);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(101, 200);
bkjm.Close();
}
/// <summary>Test speculative read feature supported by bookkeeper.</summary>
/// <remarks>
/// Test speculative read feature supported by bookkeeper. Keep one bookie
/// alive and sleep all the other bookies. Non spec client will hang for long
/// time to read the entries from the bookkeeper.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSpeculativeRead()
{
// starting 9 more servers
for (int i = 1; i < 10; i++)
{
bks.AddItem(bkutil.NewBookie());
}
NamespaceInfo nsi = NewNSInfo();
Configuration conf = new Configuration();
int ensembleSize = numLocalBookies + 9;
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100);
// sets 60 minute
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600);
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-specread"), nsi);
bkjm.Format(nsi);
long numTransactions = 1000;
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i_1 = 1; i_1 <= numTransactions; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i_1);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
// sleep 9 bk servers. Now only one server is running and responding to the
// clients
CountDownLatch sleepLatch = new CountDownLatch(1);
foreach (BookieServer bookie in bks)
{
SleepBookie(sleepLatch, bookie);
}
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
sleepLatch.CountDown();
bkjm.Close();
}
}
public virtual void Teardown()
{
if (null != zkc)
{
zkc.Close();
}
if (null != bkjm)
{
bkjm.Close();
}
}
// expected
/// <summary>Test ack quorum feature supported by bookkeeper.</summary>
/// <remarks>
/// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
/// alive and sleep all the other bookies. Now the client would wait for the
/// acknowledgement from the ack size bookies and after receiving the success
/// response will continue writing. Non ack client will hang long time to add
/// entries.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAckQuorum()
{
// slow bookie
newBookie = bkutil.NewBookie();
// make quorum size and ensemble size same to avoid the interleave writing
// of the ledger entries
int ensembleSize = numBookies + 1;
int quorumSize = numBookies + 1;
int ackSize = numBookies;
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize);
// sets 60 minutes
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600);
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
CountDownLatch sleepLatch = new CountDownLatch(1);
SleepBookie(sleepLatch, newBookie);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
int numTransactions = 100;
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
sleepLatch.CountDown();
@in[0].Close();
bkjm.Close();
}
}
public virtual void TestConcurrentFormat()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-concurrentformat");
NamespaceInfo nsi = NewNSInfo();
// populate with data first
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
for (int i = 1; i < 100 * 2; i += 2)
{
bkjm.StartLogSegment(i, NameNodeLayoutVersion.CurrentLayoutVersion);
bkjm.FinalizeLogSegment(i, i + 1);
}
bkjm.Close();
int numThreads = 40;
IList <Callable <TestBookKeeperJournalManager.ThreadStatus> > threads = new AList <Callable
<TestBookKeeperJournalManager.ThreadStatus> >();
CyclicBarrier barrier = new CyclicBarrier(numThreads);
for (int i_1 = 0; i_1 < numThreads; i_1++)
{
threads.AddItem(new _Callable_784(uri, nsi, barrier));
}
ExecutorService service = Executors.NewFixedThreadPool(numThreads);
IList <Future <TestBookKeeperJournalManager.ThreadStatus> > statuses = service.InvokeAll
(threads, 60, TimeUnit.Seconds);
int numCompleted = 0;
foreach (Future <TestBookKeeperJournalManager.ThreadStatus> s in statuses)
{
NUnit.Framework.Assert.IsTrue(s.IsDone());
NUnit.Framework.Assert.IsTrue("Thread threw invalid exception", s.Get() == TestBookKeeperJournalManager.ThreadStatus
.Completed || s.Get() == TestBookKeeperJournalManager.ThreadStatus.Goodexception
);
if (s.Get() == TestBookKeeperJournalManager.ThreadStatus.Completed)
{
numCompleted++;
}
}
Log.Info("Completed " + numCompleted + " formats");
NUnit.Framework.Assert.IsTrue("No thread managed to complete formatting", numCompleted
> 0);
}
public virtual void TestCorruptInprogressNode()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(101);
zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"),
-1);
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
try
{
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating"
+ " an empty inprogess znode");
}
catch (IOException e)
{
// correct behaviour
NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains
("has no field named"));
}
finally
{
bkjm.Close();
}
}
public TestBookKeeperJournalManager.ThreadStatus Call()
{
BookKeeperJournalManager bkjm = null;
try
{
bkjm = new BookKeeperJournalManager(TestBookKeeperJournalManager.conf, uri, nsi);
barrier.Await();
bkjm.Format(nsi);
return(TestBookKeeperJournalManager.ThreadStatus.Completed);
}
catch (IOException ioe)
{
TestBookKeeperJournalManager.Log.Info("Exception formatting ", ioe);
return(TestBookKeeperJournalManager.ThreadStatus.Goodexception);
}
catch (Exception ie)
{
TestBookKeeperJournalManager.Log.Error("Interrupted. Something is broken", ie);
Sharpen.Thread.CurrentThread().Interrupt();
return(TestBookKeeperJournalManager.ThreadStatus.Badexception);
}
catch (Exception e)
{
TestBookKeeperJournalManager.Log.Error("Some other bad exception", e);
return(TestBookKeeperJournalManager.ThreadStatus.Badexception);
}
finally
{
if (bkjm != null)
{
try
{
bkjm.Close();
}
catch (IOException ioe)
{
TestBookKeeperJournalManager.Log.Error("Error closing journal manager", ioe);
}
}
}
}
public virtual void TestRefinalizeAlreadyFinalizedInprogress()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(1);
string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100);
NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode
, null));
byte[] inprogressData = zkc.GetData(inprogressZNode, false, null);
// finalize
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode
, null));
zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode
.Persistent);
// should work fine
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
}