/// <summary>return the cachedOp, and reset it to null.</summary>
internal virtual FSEditLogOp GetCachedOp()
{
FSEditLogOp op = this.cachedOp;
cachedOp = null;
return(op);
}
public virtual void TestEmptyInprogressLedger()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-emptyInprogressLedger");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
@out.Close();
bkjm.Close();
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
for (long i_1 = 1; i_1 <= 100; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i_1);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(101, 200);
bkjm.Close();
}
public virtual void TestSimpleRead()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-simpleread"), nsi);
bkjm.Format(nsi);
long numTransactions = 10000;
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
}
}
/// <exception cref="Org.Xml.Sax.SAXException"/>
public static void WriteCachePoolInfo(ContentHandler contentHandler, CachePoolInfo
info)
{
XMLUtils.AddSaxString(contentHandler, "POOLNAME", info.GetPoolName());
string ownerName = info.GetOwnerName();
string groupName = info.GetGroupName();
long limit = info.GetLimit();
FsPermission mode = info.GetMode();
long maxRelativeExpiry = info.GetMaxRelativeExpiryMs();
if (ownerName != null)
{
XMLUtils.AddSaxString(contentHandler, "OWNERNAME", ownerName);
}
if (groupName != null)
{
XMLUtils.AddSaxString(contentHandler, "GROUPNAME", groupName);
}
if (mode != null)
{
FSEditLogOp.FsPermissionToXml(contentHandler, mode);
}
if (limit != null)
{
XMLUtils.AddSaxString(contentHandler, "LIMIT", System.Convert.ToString(limit));
}
if (maxRelativeExpiry != null)
{
XMLUtils.AddSaxString(contentHandler, "MAXRELATIVEEXPIRY", System.Convert.ToString
(maxRelativeExpiry));
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Util.XMLUtils.InvalidXmlException"/>
public static CachePoolInfo ReadCachePoolInfo(XMLUtils.Stanza st)
{
string poolName = st.GetValue("POOLNAME");
CachePoolInfo info = new CachePoolInfo(poolName);
if (st.HasChildren("OWNERNAME"))
{
info.SetOwnerName(st.GetValue("OWNERNAME"));
}
if (st.HasChildren("GROUPNAME"))
{
info.SetGroupName(st.GetValue("GROUPNAME"));
}
if (st.HasChildren("MODE"))
{
info.SetMode(FSEditLogOp.FsPermissionFromXml(st));
}
if (st.HasChildren("LIMIT"))
{
info.SetLimit(long.Parse(st.GetValue("LIMIT")));
}
if (st.HasChildren("MAXRELATIVEEXPIRY"))
{
info.SetMaxRelativeExpiryMs(long.Parse(st.GetValue("MAXRELATIVEEXPIRY")));
}
return(info);
}
/// <exception cref="System.IO.IOException"/>
public static void WriteOp(EditLogOutputStream stm, long txid)
{
FSEditLogOp op = NameNodeAdapter.CreateMkdirOp("tx " + txid);
op.SetTransactionId(txid);
stm.Write(op);
}
/// <summary>
/// Verify that the given list of streams contains exactly the range of
/// transactions specified, inclusive.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public static void VerifyEdits(IList <EditLogInputStream> streams, int firstTxnId,
int lastTxnId)
{
IEnumerator <EditLogInputStream> iter = streams.GetEnumerator();
NUnit.Framework.Assert.IsTrue(iter.HasNext());
EditLogInputStream stream = iter.Next();
for (int expected = firstTxnId; expected <= lastTxnId; expected++)
{
FSEditLogOp op = stream.ReadOp();
while (op == null)
{
NUnit.Framework.Assert.IsTrue("Expected to find txid " + expected + ", " + "but no more streams available to read from"
, iter.HasNext());
stream = iter.Next();
op = stream.ReadOp();
}
NUnit.Framework.Assert.AreEqual(FSEditLogOpCodes.OpMkdir, op.opCode);
NUnit.Framework.Assert.AreEqual(expected, op.GetTransactionId());
}
NUnit.Framework.Assert.IsNull(stream.ReadOp());
NUnit.Framework.Assert.IsFalse("Expected no more txns after " + lastTxnId + " but more streams are available"
, iter.HasNext());
}
public virtual void TestExcludeInProgressStreams()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/excludeinprogressstreams");
// Don't close the edit log once the files have been set up.
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10, false);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
// If we exclude the in-progess stream, we should only have 100 tx.
NUnit.Framework.Assert.AreEqual(100, GetNumberOfTransactions(jm, 1, false, false)
);
EditLogInputStream elis = GetJournalInputStream(jm, 90, false);
try
{
FSEditLogOp lastReadOp = null;
while ((lastReadOp = elis.ReadOp()) != null)
{
NUnit.Framework.Assert.IsTrue(lastReadOp.GetTransactionId() <= 100);
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override long ScanNextOp()
{
Preconditions.CheckState(state == EditLogFileInputStream.State.Open);
FSEditLogOp cachedNext = GetCachedOp();
return(cachedNext == null?reader.ScanOp() : cachedNext.txid);
}
public virtual void TestSimpleRecovery()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-simplerecovery"), nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.SetReadyToFlush();
@out.Flush();
@out.Abort();
@out.Close();
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.InprogressZNode(1), false));
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(1, 100), false
));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(1), false));
}
/// <exception cref="System.IO.IOException"/>
public override void Write(FSEditLogOp op)
{
writer.WriteOp(op);
if (bufCurrent.GetLength() > transmissionThreshold)
{
Transmit();
}
}
/// <summary>
/// Position the stream so that a valid operation can be read from it with
/// readOp().
/// </summary>
/// <remarks>
/// Position the stream so that a valid operation can be read from it with
/// readOp().
/// This method can be used to skip over corrupted sections of edit logs.
/// </remarks>
public virtual void Resync()
{
if (cachedOp != null)
{
return;
}
cachedOp = NextValidOp();
}
/// <summary>
/// Find the id of the last edit log transaction writen to a edit log
/// ledger.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Contrib.Bkjournal.BookKeeperJournalManager.SegmentEmptyException
/// "/>
private long RecoverLastTxId(EditLogLedgerMetadata l, bool fence)
{
LedgerHandle lh = null;
try
{
if (fence)
{
lh = bkc.OpenLedger(l.GetLedgerId(), BookKeeper.DigestType.Mac, Sharpen.Runtime.GetBytesForString
(digestpw, Charsets.Utf8));
}
else
{
lh = bkc.OpenLedgerNoRecovery(l.GetLedgerId(), BookKeeper.DigestType.Mac, Sharpen.Runtime.GetBytesForString
(digestpw, Charsets.Utf8));
}
}
catch (BKException bke)
{
throw new IOException("Exception opening ledger for " + l, bke);
}
catch (Exception ie)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException("Interrupted opening ledger for " + l, ie);
}
BookKeeperEditLogInputStream @in = null;
try
{
long lastAddConfirmed = lh.GetLastAddConfirmed();
if (lastAddConfirmed == -1)
{
throw new BookKeeperJournalManager.SegmentEmptyException();
}
@in = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed);
long endTxId = HdfsConstants.InvalidTxid;
FSEditLogOp op = @in.ReadOp();
while (op != null)
{
if (endTxId == HdfsConstants.InvalidTxid || op.GetTransactionId() == endTxId + 1)
{
endTxId = op.GetTransactionId();
}
op = @in.ReadOp();
}
return(endTxId);
}
finally
{
if (@in != null)
{
@in.Close();
}
}
}
/// <summary>Test speculative read feature supported by bookkeeper.</summary>
/// <remarks>
/// Test speculative read feature supported by bookkeeper. Keep one bookie
/// alive and sleep all the other bookies. Non spec client will hang for long
/// time to read the entries from the bookkeeper.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSpeculativeRead()
{
// starting 9 more servers
for (int i = 1; i < 10; i++)
{
bks.AddItem(bkutil.NewBookie());
}
NamespaceInfo nsi = NewNSInfo();
Configuration conf = new Configuration();
int ensembleSize = numLocalBookies + 9;
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperSpeculativeReadTimeoutMs, 100);
// sets 60 minute
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperReadEntryTimeoutSec, 3600);
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-specread"), nsi);
bkjm.Format(nsi);
long numTransactions = 1000;
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i_1 = 1; i_1 <= numTransactions; i_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i_1);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
// sleep 9 bk servers. Now only one server is running and responding to the
// clients
CountDownLatch sleepLatch = new CountDownLatch(1);
foreach (BookieServer bookie in bks)
{
SleepBookie(sleepLatch, bookie);
}
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
sleepLatch.CountDown();
bkjm.Close();
}
}
public static string GetMkdirOpPath(FSEditLogOp op)
{
if (op.opCode == FSEditLogOpCodes.OpMkdir)
{
return(((FSEditLogOp.MkdirOp)op).path);
}
else
{
return(null);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void VisitOp(FSEditLogOp op)
{
try
{
op.OutputToXml(contentHandler);
}
catch (SAXException e)
{
throw new IOException("SAX error: " + e.Message);
}
}
/// <summary>Read an operation from the stream</summary>
/// <returns>an operation from the stream or null if at end of stream</returns>
/// <exception cref="System.IO.IOException">if there is an error reading from the stream
/// </exception>
public virtual FSEditLogOp ReadOp()
{
FSEditLogOp ret;
if (cachedOp != null)
{
ret = cachedOp;
cachedOp = null;
return(ret);
}
return(NextOp());
}
// expected
/// <summary>Test ack quorum feature supported by bookkeeper.</summary>
/// <remarks>
/// Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
/// alive and sleep all the other bookies. Now the client would wait for the
/// acknowledgement from the ack size bookies and after receiving the success
/// response will continue writing. Non ack client will hang long time to add
/// entries.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAckQuorum()
{
// slow bookie
newBookie = bkutil.NewBookie();
// make quorum size and ensemble size same to avoid the interleave writing
// of the ledger entries
int ensembleSize = numBookies + 1;
int quorumSize = numBookies + 1;
int ackSize = numBookies;
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAckQuorumSize, ackSize);
// sets 60 minutes
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 3600);
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
CountDownLatch sleepLatch = new CountDownLatch(1);
SleepBookie(sleepLatch, newBookie);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
int numTransactions = 100;
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
sleepLatch.CountDown();
@in[0].Close();
bkjm.Close();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDefaultAckQuorum()
{
newBookie = bkutil.NewBookie();
int ensembleSize = numBookies + 1;
int quorumSize = numBookies + 1;
// ensure that the journal manager has to use all bookies,
// so that a failure will fail the journal manager
Configuration conf = new Configuration();
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperEnsembleSize, ensembleSize);
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperQuorumSize, quorumSize);
// sets 2 secs
conf.SetInt(BookKeeperJournalManager.BkjmBookkeeperAddEntryTimeoutSec, 2);
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-onebookiefailure"), nsi);
bkjm.Format(nsi);
CountDownLatch sleepLatch = new CountDownLatch(1);
SleepBookie(sleepLatch, newBookie);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
int numTransactions = 100;
for (long i = 1; i <= numTransactions; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
try
{
@out.Close();
bkjm.FinalizeLogSegment(1, numTransactions);
IList <EditLogInputStream> @in = new AList <EditLogInputStream>();
bkjm.SelectInputStreams(@in, 1, true);
try
{
NUnit.Framework.Assert.AreEqual(numTransactions, FSEditLogTestUtil.CountTransactionsInStream
(@in[0]));
}
finally
{
@in[0].Close();
}
NUnit.Framework.Assert.Fail("Should throw exception as not enough non-faulty bookies available!"
);
}
catch (IOException)
{
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void WriteOp(FSEditLogOp op)
{
if (firstTxId == HdfsConstants.InvalidTxid)
{
firstTxId = op.txid;
}
else
{
System.Diagnostics.Debug.Assert(op.txid > firstTxId);
}
writer.WriteOp(op);
numTxns++;
}
/// <exception cref="System.Exception"/>
public static byte[] CreateTxnData(int startTxn, int numTxns)
{
DataOutputBuffer buf = new DataOutputBuffer();
FSEditLogOp.Writer writer = new FSEditLogOp.Writer(buf);
for (long txid = startTxn; txid < startTxn + numTxns; txid++)
{
FSEditLogOp op = NameNodeAdapter.CreateMkdirOp("tx " + txid);
op.SetTransactionId(txid);
writer.WriteOp(op);
}
return(Arrays.CopyOf(buf.GetData(), buf.GetLength()));
}
/// <summary>Skip forward to specified transaction id.</summary>
/// <remarks>
/// Skip forward to specified transaction id.
/// Currently we do this by just iterating forward.
/// If this proves to be too expensive, this can be reimplemented
/// with a binary search over bk entries
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void SkipTo(long txId)
{
long numToSkip = GetFirstTxId() - txId;
FSEditLogOp op = null;
for (long i = 0; i < numToSkip; i++)
{
op = ReadOp();
}
if (op != null && op.GetTransactionId() != txId - 1)
{
throw new IOException("Corrupt stream, expected txid " + (txId - 1) + ", got " +
op.GetTransactionId());
}
}
/// <summary>
/// Skip edit log operations up to a given transaction ID, or until the
/// end of the edit log is reached.
/// </summary>
/// <remarks>
/// Skip edit log operations up to a given transaction ID, or until the
/// end of the edit log is reached.
/// After this function returns, the next call to readOp will return either
/// end-of-file (null) or a transaction with a txid equal to or higher than
/// the one we asked for.
/// </remarks>
/// <param name="txid">The transaction ID to read up until.</param>
/// <returns>
/// Returns true if we found a transaction ID greater than
/// or equal to 'txid' in the log.
/// </returns>
/// <exception cref="System.IO.IOException"/>
public virtual bool SkipUntil(long txid)
{
while (true)
{
FSEditLogOp op = ReadOp();
if (op == null)
{
return(false);
}
if (op.GetTransactionId() >= txid)
{
cachedOp = op;
return(true);
}
}
}
/// <summary>
/// Find out how many transactions we can read from a
/// FileJournalManager, starting at a given transaction ID.
/// </summary>
/// <param name="jm">The journal manager</param>
/// <param name="fromTxId">Transaction ID to start at</param>
/// <param name="inProgressOk">Should we consider edit logs that are not finalized?</param>
/// <returns>The number of transactions</returns>
/// <exception cref="System.IO.IOException"/>
internal static long GetNumberOfTransactions(FileJournalManager jm, long fromTxId
, bool inProgressOk, bool abortOnGap)
{
long numTransactions = 0;
long txId = fromTxId;
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
jm.SelectInputStreams(allStreams, fromTxId, inProgressOk);
EditLogInputStream elis = null;
try
{
while ((elis = allStreams.Poll()) != null)
{
try
{
elis.SkipUntil(txId);
while (true)
{
FSEditLogOp op = elis.ReadOp();
if (op == null)
{
break;
}
if (abortOnGap && (op.GetTransactionId() != txId))
{
Log.Info("getNumberOfTransactions: detected gap at txId " + fromTxId);
return(numTransactions);
}
txId = op.GetTransactionId() + 1;
numTransactions++;
}
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(allStreams, new EditLogInputStream
[0]));
}
return(numTransactions);
}
public virtual void TestCorruptInprogressNode()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-corruptInprogress");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(1, 100);
@out = bkjm.StartLogSegment(101, NameNodeLayoutVersion.CurrentLayoutVersion);
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(101);
zkc.SetData(inprogressZNode, Sharpen.Runtime.GetBytesForString("WholeLottaJunk"),
-1);
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
try
{
bkjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed. There should be no way of creating"
+ " an empty inprogess znode");
}
catch (IOException e)
{
// correct behaviour
NUnit.Framework.Assert.IsTrue("Exception different than expected", e.Message.Contains
("has no field named"));
}
finally
{
bkjm.Close();
}
}
public virtual void TestNumberOfTransactionsWithInprogressAtEnd()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-inprogressAtEnd"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, (txid - 1));
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, (txid
- 1)), false));
}
long start_1 = txid;
EditLogOutputStream out_1 = bkjm.StartLogSegment(start_1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j_1 = 1; j_1 <= DefaultSegmentSize / 2; j_1++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
out_1.Write(op);
}
out_1.SetReadyToFlush();
out_1.Flush();
out_1.Abort();
out_1.Close();
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual((txid - 1), numTrans);
}
public virtual void TestNumberOfTransactionsWithGaps()
{
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.CreateJournalURI
("/hdfsjournal-gaps"), nsi);
bkjm.Format(nsi);
long txid = 1;
for (long i = 0; i < 3; i++)
{
long start = txid;
EditLogOutputStream @out = bkjm.StartLogSegment(start, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long j = 1; j <= DefaultSegmentSize; j++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(txid++);
@out.Write(op);
}
@out.Close();
bkjm.FinalizeLogSegment(start, txid - 1);
NUnit.Framework.Assert.IsNotNull(zkc.Exists(bkjm.FinalizedLedgerZNode(start, txid
- 1), false));
}
zkc.Delete(bkjm.FinalizedLedgerZNode(DefaultSegmentSize + 1, DefaultSegmentSize *
2), -1);
long numTrans = bkjm.GetNumberOfTransactions(1, true);
NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans);
try
{
numTrans = bkjm.GetNumberOfTransactions(DefaultSegmentSize + 1, true);
NUnit.Framework.Assert.Fail("Should have thrown corruption exception by this point"
);
}
catch (JournalManager.CorruptionException)
{
}
// if we get here, everything is going good
numTrans = bkjm.GetNumberOfTransactions((DefaultSegmentSize * 2) + 1, true);
NUnit.Framework.Assert.AreEqual(DefaultSegmentSize, numTrans);
}
public virtual void TestRefinalizeAlreadyFinalizedInprogress()
{
URI uri = BKJMUtil.CreateJournalURI("/hdfsjournal-refinalizeInprogressLedger");
NamespaceInfo nsi = NewNSInfo();
BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.Format(nsi);
EditLogOutputStream @out = bkjm.StartLogSegment(1, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (long i = 1; i <= 100; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
bkjm.Close();
string inprogressZNode = bkjm.InprogressZNode(1);
string finalizedZNode = bkjm.FinalizedLedgerZNode(1, 100);
NUnit.Framework.Assert.IsNotNull("inprogress znode doesn't exist", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNull("finalized znode exists", zkc.Exists(finalizedZNode
, null));
byte[] inprogressData = zkc.GetData(inprogressZNode, false, null);
// finalize
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
NUnit.Framework.Assert.IsNull("inprogress znode exists", zkc.Exists(inprogressZNode
, null));
NUnit.Framework.Assert.IsNotNull("finalized znode doesn't exist", zkc.Exists(finalizedZNode
, null));
zkc.Create(inprogressZNode, inprogressData, ZooDefs.Ids.OpenAclUnsafe, CreateMode
.Persistent);
// should work fine
bkjm = new BookKeeperJournalManager(conf, uri, nsi);
bkjm.RecoverUnfinalizedSegments();
bkjm.Close();
}
public virtual void TestReadFromMiddleOfEditLog()
{
FilePath f = new FilePath(TestEditLog.TestDir + "/readfrommiddleofeditlog");
NNStorage storage = TestEditLog.SetupEdits(Sharpen.Collections.SingletonList <URI>
(f.ToURI()), 10);
Storage.StorageDirectory sd = storage.DirIterator(NNStorage.NameNodeDirType.Edits
).Next();
FileJournalManager jm = new FileJournalManager(conf, sd, storage);
EditLogInputStream elis = GetJournalInputStream(jm, 5, true);
try
{
FSEditLogOp op = elis.ReadOp();
NUnit.Framework.Assert.AreEqual("read unexpected op", op.GetTransactionId(), 5);
}
finally
{
IOUtils.Cleanup(Log, elis);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Zookeeper.KeeperException"/>
/// <exception cref="System.Exception"/>
private string StartAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid
, int endTxid)
{
EditLogOutputStream @out = bkjm.StartLogSegment(startTxid, NameNodeLayoutVersion.
CurrentLayoutVersion);
for (long i = startTxid; i <= endTxid; i++)
{
FSEditLogOp op = FSEditLogTestUtil.GetNoOpInstance();
op.SetTransactionId(i);
@out.Write(op);
}
@out.Close();
// finalize the inprogress_1 log segment.
bkjm.FinalizeLogSegment(startTxid, endTxid);
string zkpath1 = bkjm.FinalizedLedgerZNode(startTxid, endTxid);
NUnit.Framework.Assert.IsNotNull(zkc.Exists(zkpath1, false));
NUnit.Framework.Assert.IsNull(zkc.Exists(bkjm.InprogressZNode(startTxid), false));
return(zkpath1);
}
