/// <summary> /// Find the id of the last edit log transaction writen to a edit log /// ledger. /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Contrib.Bkjournal.BookKeeperJournalManager.SegmentEmptyException /// "/> private long RecoverLastTxId(EditLogLedgerMetadata l, bool fence) { LedgerHandle lh = null; try { if (fence) { lh = bkc.OpenLedger(l.GetLedgerId(), BookKeeper.DigestType.Mac, Sharpen.Runtime.GetBytesForString (digestpw, Charsets.Utf8)); } else { lh = bkc.OpenLedgerNoRecovery(l.GetLedgerId(), BookKeeper.DigestType.Mac, Sharpen.Runtime.GetBytesForString (digestpw, Charsets.Utf8)); } } catch (BKException bke) { throw new IOException("Exception opening ledger for " + l, bke); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Interrupted opening ledger for " + l, ie); } BookKeeperEditLogInputStream @in = null; try { long lastAddConfirmed = lh.GetLastAddConfirmed(); if (lastAddConfirmed == -1) { throw new BookKeeperJournalManager.SegmentEmptyException(); } @in = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed); long endTxId = HdfsConstants.InvalidTxid; FSEditLogOp op = @in.ReadOp(); while (op != null) { if (endTxId == HdfsConstants.InvalidTxid || op.GetTransactionId() == endTxId + 1) { endTxId = op.GetTransactionId(); } op = @in.ReadOp(); } return(endTxId); } finally { if (@in != null) { @in.Close(); } } }
/// <summary>Construct an edit log output stream which writes to a ledger.</summary> /// <exception cref="System.IO.IOException"/> protected internal BookKeeperEditLogOutputStream(Configuration conf, LedgerHandle lh) : base() { bufCurrent = new DataOutputBuffer(); outstandingRequests = new AtomicInteger(0); syncLatch = null; this.lh = lh; this.writer = new FSEditLogOp.Writer(bufCurrent); this.transmissionThreshold = conf.GetInt(BookKeeperJournalManager.BkjmOutputBufferSize , BookKeeperJournalManager.BkjmOutputBufferSizeDefault); }
private void CleanupLedger(LedgerHandle lh) { try { long id = currentLedger.GetId(); currentLedger.Close(); bkc.DeleteLedger(id); } catch (BKException bke) { //log & ignore, an IOException will be thrown soon Log.Error("Error closing ledger", bke); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); Log.Warn("Interrupted while closing ledger", ie); } }
public virtual void AddComplete(int rc, LedgerHandle handle, long entryId, object ctx) { lock (this) { outstandingRequests.DecrementAndGet(); if (!transmitResult.CompareAndSet(BKException.Code.Ok, rc)) { Log.Warn("Tried to set transmit result to (" + rc + ") \"" + BKException.GetMessage (rc) + "\"" + " but is already (" + transmitResult.Get() + ") \"" + BKException. GetMessage(transmitResult.Get()) + "\""); } CountDownLatch l = syncLatch; if (l != null) { l.CountDown(); } } }
public virtual void TestEmptyInputStream() { ZooKeeper zk = BKJMUtil.ConnectZooKeeper(); BookKeeper bkc = new BookKeeper(new ClientConfiguration(), zk); try { LedgerHandle lh = bkc.CreateLedger(BookKeeper.DigestType.Crc32, Sharpen.Runtime.GetBytesForString ("foobar")); lh.Close(); EditLogLedgerMetadata metadata = new EditLogLedgerMetadata("/foobar", HdfsConstants .NamenodeLayoutVersion, lh.GetId(), unchecked ((int)(0x1234))); try { new BookKeeperEditLogInputStream(lh, metadata, -1); NUnit.Framework.Assert.Fail("Shouldn't get this far, should have thrown"); } catch (IOException ioe) { NUnit.Framework.Assert.IsTrue(ioe.Message.Contains("Invalid first bk entry to read" )); } metadata = new EditLogLedgerMetadata("/foobar", HdfsConstants.NamenodeLayoutVersion , lh.GetId(), unchecked ((int)(0x1234))); try { new BookKeeperEditLogInputStream(lh, metadata, 0); NUnit.Framework.Assert.Fail("Shouldn't get this far, should have thrown"); } catch (IOException ioe) { NUnit.Framework.Assert.IsTrue(ioe.Message.Contains("Invalid first bk entry to read" )); } } finally { bkc.Close(); zk.Close(); } }
/// <summary>Construct BookKeeper edit log input stream.</summary> /// <remarks> /// Construct BookKeeper edit log input stream. /// Starts reading from firstBookKeeperEntry. This allows the stream /// to take a shortcut during recovery, as it doesn't have to read /// every edit log transaction to find out what the last one is. /// </remarks> /// <exception cref="System.IO.IOException"/> internal BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata , long firstBookKeeperEntry) { this.lh = lh; this.firstTxId = metadata.GetFirstTxId(); this.lastTxId = metadata.GetLastTxId(); this.logVersion = metadata.GetDataLayoutVersion(); this.inProgress = metadata.IsInProgress(); if (firstBookKeeperEntry < 0 || firstBookKeeperEntry > lh.GetLastAddConfirmed()) { throw new IOException("Invalid first bk entry to read: " + firstBookKeeperEntry + ", LAC: " + lh.GetLastAddConfirmed()); } BufferedInputStream bin = new BufferedInputStream(new BookKeeperEditLogInputStream.LedgerInputStream (lh, firstBookKeeperEntry)); tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); DataInputStream @in = new DataInputStream(tracker); reader = new FSEditLogOp.Reader(@in, tracker, logVersion); }
/// <summary>Finalize a log segment.</summary> /// <remarks> /// Finalize a log segment. If the journal manager is currently /// writing to a ledger, ensure that this is the ledger of the log segment /// being finalized. /// Otherwise this is the recovery case. In the recovery case, ensure that /// the firstTxId of the ledger matches firstTxId for the segment we are /// trying to finalize. /// </remarks> /// <exception cref="System.IO.IOException"/> public override void FinalizeLogSegment(long firstTxId, long lastTxId) { CheckEnv(); string inprogressPath = InprogressZNode(firstTxId); try { Stat inprogressStat = zkc.Exists(inprogressPath, false); if (inprogressStat == null) { throw new IOException("Inprogress znode " + inprogressPath + " doesn't exist"); } EditLogLedgerMetadata l = EditLogLedgerMetadata.Read(zkc, inprogressPath); if (currentLedger != null) { // normal, non-recovery case if (l.GetLedgerId() == currentLedger.GetId()) { try { currentLedger.Close(); } catch (BKException bke) { Log.Error("Error closing current ledger", bke); } currentLedger = null; } else { throw new IOException("Active ledger has different ID to inprogress. " + l.GetLedgerId () + " found, " + currentLedger.GetId() + " expected"); } } if (l.GetFirstTxId() != firstTxId) { throw new IOException("Transaction id not as expected, " + l.GetFirstTxId() + " found, " + firstTxId + " expected"); } l.FinalizeLedger(lastTxId); string finalisedPath = FinalizedLedgerZNode(firstTxId, lastTxId); try { l.Write(zkc, finalisedPath); } catch (KeeperException.NodeExistsException) { if (!l.Verify(zkc, finalisedPath)) { throw new IOException("Node " + finalisedPath + " already exists" + " but data doesn't match" ); } } maxTxId.Store(lastTxId); zkc.Delete(inprogressPath, inprogressStat.GetVersion()); string inprogressPathFromCI = ci.Read(); if (inprogressPath.Equals(inprogressPathFromCI)) { ci.Clear(); } } catch (KeeperException e) { throw new IOException("Error finalising ledger", e); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Error finalising ledger", ie); } }
/// <summary>Start a new log segment in a BookKeeper ledger.</summary> /// <remarks> /// Start a new log segment in a BookKeeper ledger. /// First ensure that we have the write lock for this journal. /// Then create a ledger and stream based on that ledger. /// The ledger id is written to the inprogress znode, so that in the /// case of a crash, a recovery process can find the ledger we were writing /// to when we crashed. /// </remarks> /// <param name="txId">First transaction id to be written to the stream</param> /// <exception cref="System.IO.IOException"/> public override EditLogOutputStream StartLogSegment(long txId, int layoutVersion) { CheckEnv(); if (txId <= maxTxId.Get()) { throw new IOException("We've already seen " + txId + ". A new stream cannot be created with it" ); } try { string existingInprogressNode = ci.Read(); if (null != existingInprogressNode && zkc.Exists(existingInprogressNode, false) != null) { throw new IOException("Inprogress node already exists"); } if (currentLedger != null) { // bookkeeper errored on last stream, clean up ledger currentLedger.Close(); } currentLedger = bkc.CreateLedger(ensembleSize, quorumSize, ackQuorumSize, BookKeeper.DigestType .Mac, Sharpen.Runtime.GetBytesForString(digestpw, Charsets.Utf8)); } catch (BKException bke) { throw new IOException("Error creating ledger", bke); } catch (KeeperException ke) { throw new IOException("Error in zookeeper while creating ledger", ke); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Interrupted creating ledger", ie); } try { string znodePath = InprogressZNode(txId); EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath, layoutVersion, currentLedger .GetId(), txId); /* Write the ledger metadata out to the inprogress ledger znode * This can fail if for some reason our write lock has * expired (@see WriteLock) and another process has managed to * create the inprogress znode. * In this case, throw an exception. We don't want to continue * as this would lead to a split brain situation. */ l.Write(zkc, znodePath); maxTxId.Store(txId); ci.Update(znodePath); return(new BookKeeperEditLogOutputStream(conf, currentLedger)); } catch (KeeperException ke) { CleanupLedger(currentLedger); throw new IOException("Error storing ledger metadata", ke); } }
/// <summary>Construct BookKeeper edit log input stream.</summary> /// <remarks> /// Construct BookKeeper edit log input stream. /// Starts reading from the first entry of the ledger. /// </remarks> /// <exception cref="System.IO.IOException"/> internal BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata ) : this(lh, metadata, 0) { }
/// <summary>Construct ledger input stream</summary> /// <param name="lh">the ledger handle to read from</param> /// <param name="firstBookKeeperEntry">ledger entry to start reading from</param> /// <exception cref="System.IO.IOException"/> internal LedgerInputStream(LedgerHandle lh, long firstBookKeeperEntry) { this.lh = lh; readEntries = firstBookKeeperEntry; maxEntry = lh.GetLastAddConfirmed(); }