public virtual void TestWriteEdits()
{
EditLogOutputStream stm = CreateLogSegment();
QJMTestUtil.WriteOp(stm, 1);
QJMTestUtil.WriteOp(stm, 2);
stm.SetReadyToFlush();
QJMTestUtil.WriteOp(stm, 3);
// The flush should log txn 1-2
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
// Another flush should now log txn #3
stm.SetReadyToFlush();
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
}
/// <exception cref="System.IO.IOException"/>
public static void WriteTxns(EditLogOutputStream stm, long startTxId, int numTxns
)
{
for (long txid = startTxId; txid < startTxId + numTxns; txid++)
{
WriteOp(stm, txid);
}
stm.SetReadyToFlush();
stm.Flush();
}
/// <summary>Test speculative read feature supported by bookkeeper.</summary>
/// <remarks>
/// Test speculative read feature supported by bookkeeper. Keep one bookie
/// alive and sleep all the other bookies. Non spec client will hang for long
/// time to read the entries from the bookkeeper.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSpeculativeRead()
{
// starting 9 more servers
for (int i = 1; i < 10; i++)
{
bks.AddItem(bkutil.NewBookie());
}
NamespaceInfo nsi = NewNSInfo();
Configuration conf = new Configuration();
int ensembleSize = numLocalBookies + 9;
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100);
// sets 60 minute
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600);
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-specread"), nsi);
bkjm.Format(nsi);
long numTransactions = 1000;
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i_1 = 1; i_1 <= numTransactions; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i_1);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
// sleep 9 bk servers. Now only one server is running and responding to the
// clients
CountDownLatch sleepLatch = new CountDownLatch(1);
foreach (BookieServer bookie in bks)
{
SleepBookie(sleepLatch, bookie);
}
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
sleepLatch.CountDown();
bkjm.Close();
}
}
// expected
/// <summary>Test ack quorum feature supported by bookkeeper.</summary>
/// <remarks>
/// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
/// alive and sleep all the other bookies. Now the client would wait for the
/// acknowledgement from the ack size bookies and after receiving the success
/// response will continue writing. Non ack client will hang long time to add
/// entries.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAckQuorum()
{
// slow bookie
newBookie = bkutil.NewBookie();
// make quorum size and ensemble size same to avoid the interleave writing
// of the ledger entries
int ensembleSize = numBookies + 1;
int quorumSize = numBookies + 1;
int ackSize = numBookies;
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize);
// sets 60 minutes
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600);
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
CountDownLatch sleepLatch = new CountDownLatch(1);
SleepBookie(sleepLatch, newBookie);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
int numTransactions = 100;
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
sleepLatch.CountDown();
@in[0].Close();
bkjm.Close();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDefaultAckQuorum()
{
newBookie = bkutil.NewBookie();
int ensembleSize = numBookies + 1;
int quorumSize = numBookies + 1;
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
// sets 2 secs
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 2);
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
CountDownLatch sleepLatch = new CountDownLatch(1);
SleepBookie(sleepLatch, newBookie);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
int numTransactions = 100;
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
try
{
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
}
NUnit.Framework.Assert.Fail("Should throw exception as not enough non-faulty bookies available!"
);
}
catch (IOException)
{
}
}
/// <exception cref="System.IO.IOException"/>
private EditLogOutputStream CreateLogSegment()
{
FutureReturns(null).When(spyLoggers[0]).StartLogSegment(Org.Mockito.Mockito.AnyLong
(), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion.CurrentLayoutVersion));
FutureReturns(null).When(spyLoggers[1]).StartLogSegment(Org.Mockito.Mockito.AnyLong
(), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion.CurrentLayoutVersion));
FutureReturns(null).When(spyLoggers[2]).StartLogSegment(Org.Mockito.Mockito.AnyLong
(), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion.CurrentLayoutVersion));
EditLogOutputStream stm = qjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
return(stm);
}
public virtual void TestCorruptInprogressNode()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(101);
zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"),
-1);
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
try
{
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating"
+ " an empty inprogess znode");
}
catch (IOException e)
{
// correct behaviour
NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains
("has no field named"));
}
finally
{
bkjm.Close();
}
}
public virtual void TestNumberOfTransactionsWithGaps()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-gaps"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, txid - 1);
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, txid
- 1), false));
}
zkc.Delete(bkjm.FinalizedLedgerZNode(DefaultSegmentSize + 1, DefaultSegmentSize *
2), -1);
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans);
try
{
numTrans = bkjm.GetNumberOfTransactions(DefaultSegmentSize + 1, true);
NUnit.Framework.Assert.Fail("Should have thrown corruption exception by this point"
);
}
catch (JournalManager.CorruptionException)
{
}
// if we get here, everything is going good
numTrans = bkjm.GetNumberOfTransactions((DefaultSegmentSize * 2) + 1, true);
NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans);
}
public virtual void TestWriteEditsOneSlow()
{
EditLogOutputStream stm = CreateLogSegment();
QJMTestUtil.WriteOp(stm, 1);
stm.SetReadyToFlush();
// Make the first two logs respond immediately
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
// And the third log not respond
SettableFuture <Void> slowLog = SettableFuture.Create();
Org.Mockito.Mockito.DoReturn(slowLog).When(spyLoggers[2]).SendEdits(Matchers.AnyLong
(), Matchers.Eq(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
Org.Mockito.Mockito.Verify(spyLoggers[0]).SetCommittedTxId(1L);
}
public virtual void TestNumberOfTransactionsWithInprogressAtEnd()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-inprogressAtEnd"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid
- 1)), false));
}
long start_1 = txid;
EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
out_1.Write(op);
}
out_1.SetReadyToFlush();
out_1.Flush();
out_1.Abort();
out_1.Close();
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual((txid - 1), numTrans);
}
/// <exception cref="System.IO.IOException"/>
public static EditLogOutputStream WriteSegment(MiniJournalCluster cluster, QuorumJournalManager
qjm, long startTxId, int numTxns, bool finalize)
{
EditLogOutputStream stm = qjm.StartLogSegment(startTxId, NameNodeLayoutVersion.CurrentLayoutVersion
);
// Should create in-progress
AssertExistsInQuorum(cluster, NNStorage.GetInProgressEditsFileName(startTxId));
WriteTxns(stm, startTxId, numTxns);
if (finalize)
{
stm.Close();
qjm.FinalizeLogSegment(startTxId, startTxId + numTxns - 1);
return(null);
}
else
{
return(stm);
}
}
public virtual void TestRefinalizeAlreadyFinalizedInprogress()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(1);
string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100);
NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode
, null));
byte[] inprogressData = zkc.GetData(inprogressZNode, false, null);
// finalize
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode
, null));
zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode
.Persistent);
// should work fine
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
}
public virtual void TestMissFinalizeAndNextStart()
{
// Logger 0: miss finalize(1-3) and start(4)
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(3L));
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).StartLogSegment(Org.Mockito.Mockito.Eq(4L), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion
.CurrentLayoutVersion));
// Logger 1: fail at txn id 4
FailLoggerAtTxn(spies[1], 4L);
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail to write");
}
catch (QuorumException qe)
{
// Should fail, because logger 1 had an injected fault and
// logger 0 should detect writer out of sync
GenericTestUtils.AssertExceptionContains("Writer out of sync", qe);
}
finally
{
stm.Abort();
qjm.Close();
}
// State:
// Logger 0: 1-3 in-progress (since it missed finalize)
// Logger 1: 1-3 finalized
// Logger 2: 1-3 finalized, 4 in-progress with one txn
// Shut down logger 2 so it doesn't participate in recovery
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
long recovered = QJMTestUtil.RecoverAndReturnLastTxn(qjm);
NUnit.Framework.Assert.AreEqual(3L, recovered);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Zookeeper.KeeperException"/>
/// <exception cref="System.Exception"/>
private string StartAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid
, int endTxid)
{
EditLogOutputStream @out = bkjm.StartLogSegment(startTxid, NameNodeLayoutVersion.
CurrentLayoutVersion);
for (long i = startTxid; i <= endTxid; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
// finalize the inprogress_1 log segment.
bkjm.FinalizeLogSegment(startTxid, endTxid);
string zkpath1 = bkjm.FinalizedLedgerZNode(startTxid, endTxid);
NUnit.Framework.Assert.IsNotNull(zkc.Exists(zkpath1, false));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(startTxid), false));
return(zkpath1);
}
public virtual void TestCrashAtBeginningOfSegment()
{
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
}
finally
{
stm.Abort();
}
// Make a new QJM
qjm = CloseLater(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil
.Jid), QJMTestUtil.FakeNsinfo));
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 3);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
}
public virtual void TestNumberOfTransactions()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-txncount"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual(100, numTrans);
}
public virtual void TestSimpleWrite()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-simplewrite"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
string zkpath = bkjm.FinalizedLedgerZNode(1, 100);
NUnit.Framework.Assert.IsNotNull(zkc.Exists(zkpath, false));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false));
}
/// <summary>
/// Set up the loggers into the following state:
/// - JN0: edits 1-3 in progress
/// - JN1: edits 1-4 in progress
/// - JN2: edits 1-5 in progress
/// None of the loggers have any associated paxos info.
/// </summary>
/// <exception cref="System.Exception"/>
private void SetupLoggers345()
{
EditLogOutputStream stm = qjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
FailLoggerAtTxn(spies[0], 4);
FailLoggerAtTxn(spies[1], 5);
QJMTestUtil.WriteTxns(stm, 1, 3);
// This should succeed to 2/3 loggers
QJMTestUtil.WriteTxns(stm, 4, 1);
// This should only succeed to 1 logger (index 2). Hence it should
// fail
try
{
QJMTestUtil.WriteTxns(stm, 5, 1);
NUnit.Framework.Assert.Fail("Did not fail to write when only a minority succeeded"
);
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("too many exceptions to achieve quorum size 2/3"
, qe);
}
}
private long WriteSegmentUntilCrash(MiniJournalCluster cluster, QuorumJournalManager
qjm, long txid, int numTxns, Holder <Exception> thrown)
{
long firstTxId = txid;
long lastAcked = txid - 1;
try
{
EditLogOutputStream stm = qjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (int i = 0; i < numTxns; i++)
{
QJMTestUtil.WriteTxns(stm, txid++, 1);
lastAcked++;
}
stm.Close();
qjm.FinalizeLogSegment(firstTxId, lastAcked);
}
catch (Exception t)
{
thrown.held = t;
}
return(lastAcked);
}
public virtual void TestWriteRestartFrom1()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-restartFrom1"), nsi);
bkjm.Format(nsi);
long txid = 1;
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
txid = 1;
try
{
@out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.Fail("Shouldn't be able to start another journal from " +
txid + " when one already exists");
}
catch (Exception ioe)
{
Log.Info("Caught exception as expected", ioe);
}
// test border case
txid = DefaultSegmentSize;
try
{
@out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.Fail("Shouldn't be able to start another journal from " +
txid + " when one already exists");
}
catch (IOException ioe)
{
Log.Info("Caught exception as expected", ioe);
}
// open journal continuing from before
txid = DefaultSegmentSize + 1;
start = txid;
@out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.IsNotNull(@out);
for (long j_1 = 1; j_1 <= DefaultSegmentSize; j_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
// open journal arbitarily far in the future
txid = DefaultSegmentSize * 4;
@out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion);
NUnit.Framework.Assert.IsNotNull(@out);
}
public virtual void TestOneBookieFailure()
{
newBookie = bkutil.NewBookie();
BookieServer replacementBookie = null;
try
{
int ensembleSize = numBookies + 1;
NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp
(ensembleSize, 10));
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
long txid = 1;
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 3; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
replacementBookie = bkutil.NewBookie();
NUnit.Framework.Assert.AreEqual("replacement bookie didn't start", ensembleSize +
1, bkutil.CheckBookiesUp(ensembleSize + 1, 10));
newBookie.Shutdown();
NUnit.Framework.Assert.AreEqual("New bookie didn't die", ensembleSize, bkutil.CheckBookiesUp
(ensembleSize, 10));
for (long i_1 = 1; i_1 <= 3; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
}
catch (Exception e)
{
Log.Error("Exception in test", e);
throw;
}
finally
{
if (replacementBookie != null)
{
replacementBookie.Shutdown();
}
newBookie.Shutdown();
if (bkutil.CheckBookiesUp(numBookies, 30) != numBookies)
{
Log.Warn("Not all bookies from this test shut down, expect errors");
}
}
}
/// <summary>
/// Test the case where, at the beginning of a segment, transactions
/// have been written to one JN but not others.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void DoTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
{
int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
cluster.GetJournalNode(nodeMissingSegment).StopAndJoin(0);
// Open segment on 2/3 nodes
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
// Write transactions to only 1/3 nodes
FailLoggerAtTxn(spies[nodeWithEmptySegment], 4);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail even though 2/3 failed");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
}
finally
{
stm.Abort();
}
// Bring back the down JN.
cluster.RestartJournalNode(nodeMissingSegment);
// Make a new QJM. At this point, the state is as follows:
// A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)
// B: nodeWithOneTxn: 1-3 finalized, 4_inprogress (1 txn)
// C: nodeMissingSegment: 1-3 finalized
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithEmptySegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeWithOneTxn, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3), NNStorage.GetInProgressEditsFileName
(4));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(nodeMissingSegment, QJMTestUtil
.Jid), "edits_.*", NNStorage.GetFinalizedEditsFileName(1, 3));
// Stop one of the nodes. Since we run this test three
// times, rotating the roles of the nodes, we'll test
// all the permutations.
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
qjm.RecoverUnfinalizedSegments();
if (nodeWithOneTxn == 0 || nodeWithOneTxn == 1)
{
// If the node that had the transaction committed was one of the nodes
// that responded during recovery, then we should have recovered txid
// 4.
CheckRecovery(cluster, 4, 4);
QJMTestUtil.WriteSegment(cluster, qjm, 5, 3, true);
}
else
{
// Otherwise, we should have recovered only 1-3 and should be able to
// start a segment at 4.
CheckRecovery(cluster, 1, 3);
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
}
}
