public virtual void TestWriteEdits()
{
EditLogOutputStream stm = CreateLogSegment();
QJMTestUtil.WriteOp(stm, 1);
QJMTestUtil.WriteOp(stm, 2);
stm.SetReadyToFlush();
QJMTestUtil.WriteOp(stm, 3);
// The flush should log txn 1-2
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(2), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
// Another flush should now log txn #3
stm.SetReadyToFlush();
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[2]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(3L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
}
public virtual void TestSimpleRecovery()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-simplerecovery"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
@out.Abort();
@out.Close();
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false));
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false));
}
/// <exception cref="System.IO.IOException"/>
public static void WriteTxns(EditLogOutputStream stm, long startTxId, int numTxns
)
{
for (long txid = startTxId; txid < startTxId + numTxns; txid++)
{
WriteOp(stm, txid);
}
stm.SetReadyToFlush();
stm.Flush();
}
public virtual void TestNumberOfTransactionsWithInprogressAtEnd()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-inprogressAtEnd"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid
- 1)), false));
}
long start_1 = txid;
EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
out_1.Write(op);
}
out_1.SetReadyToFlush();
out_1.Flush();
out_1.Abort();
out_1.Close();
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual((txid - 1), numTrans);
}
public virtual void TestWriteEditsOneSlow()
{
EditLogOutputStream stm = CreateLogSegment();
QJMTestUtil.WriteOp(stm, 1);
stm.SetReadyToFlush();
// Make the first two logs respond immediately
FutureReturns(null).When(spyLoggers[0]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
FutureReturns(null).When(spyLoggers[1]).SendEdits(Matchers.AnyLong(), Matchers.Eq
(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
// And the third log not respond
SettableFuture <Void> slowLog = SettableFuture.Create();
Org.Mockito.Mockito.DoReturn(slowLog).When(spyLoggers[2]).SendEdits(Matchers.AnyLong
(), Matchers.Eq(1L), Matchers.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
stm.Flush();
Org.Mockito.Mockito.Verify(spyLoggers[0]).SetCommittedTxId(1L);
}
public virtual void TestOneBookieFailure()
{
newBookie = bkutil.NewBookie();
BookieServer replacementBookie = null;
try
{
int ensembleSize = numBookies + 1;
NUnit.Framework.Assert.AreEqual("New bookie didn't start", ensembleSize, bkutil.CheckBookiesUp
(ensembleSize, 10));
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
long txid = 1;
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 3; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
replacementBookie = bkutil.NewBookie();
NUnit.Framework.Assert.AreEqual("replacement bookie didn't start", ensembleSize +
1, bkutil.CheckBookiesUp(ensembleSize + 1, 10));
newBookie.Shutdown();
NUnit.Framework.Assert.AreEqual("New bookie didn't die", ensembleSize, bkutil.CheckBookiesUp
(ensembleSize, 10));
for (long i_1 = 1; i_1 <= 3; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
}
catch (Exception e)
{
Log.Error("Exception in test", e);
throw;
}
finally
{
if (replacementBookie != null)
{
replacementBookie.Shutdown();
}
newBookie.Shutdown();
if (bkutil.CheckBookiesUp(numBookies, 30) != numBookies)
{
Log.Warn("Not all bookies from this test shut down, expect errors");
}
}
}