/// <exception cref="System.IO.IOException"/> public override void RecoverUnfinalizedSegments() { CheckEnv(); lock (this) { try { IList <string> children = zkc.GetChildren(ledgerPath, false); foreach (string child in children) { if (!child.StartsWith(BkjmEditInprogress)) { continue; } string znode = ledgerPath + "/" + child; EditLogLedgerMetadata l = EditLogLedgerMetadata.Read(zkc, znode); try { long endTxId = RecoverLastTxId(l, true); if (endTxId == HdfsConstants.InvalidTxid) { Log.Error("Unrecoverable corruption has occurred in segment " + l.ToString() + " at path " + znode + ". Unable to continue recovery."); throw new IOException("Unrecoverable corruption," + " please check logs."); } FinalizeLogSegment(l.GetFirstTxId(), endTxId); } catch (BookKeeperJournalManager.SegmentEmptyException) { Log.Warn("Inprogress znode " + child + " refers to a ledger which is empty. This occurs when the NN" + " crashes after opening a segment, but before writing the" + " OP_START_LOG_SEGMENT op. It is safe to delete." + " MetaData [" + l.ToString() + "]"); // If the max seen transaction is the same as what would // have been the first transaction of the failed ledger, // decrement it, as that transaction never happened and as // such, is _not_ the last seen if (maxTxId.Get() == l.GetFirstTxId()) { maxTxId.Reset(maxTxId.Get() - 1); } zkc.Delete(znode, -1); } } } catch (KeeperException.NoNodeException) { } catch (KeeperException ke) { // nothing to recover, ignore throw new IOException("Couldn't get list of inprogress segments", ke); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Interrupted getting list of inprogress segments", ie); } } }
/// <exception cref="System.IO.IOException"/> private IList <EditLogLedgerMetadata> GetLedgerList(long fromTxId, bool inProgressOk ) { IList <EditLogLedgerMetadata> ledgers = new AList <EditLogLedgerMetadata>(); try { IList <string> ledgerNames = zkc.GetChildren(ledgerPath, false); foreach (string ledgerName in ledgerNames) { if (!inProgressOk && ledgerName.Contains(BkjmEditInprogress)) { continue; } string legderMetadataPath = ledgerPath + "/" + ledgerName; try { EditLogLedgerMetadata editLogLedgerMetadata = EditLogLedgerMetadata.Read(zkc, legderMetadataPath ); if (editLogLedgerMetadata.GetLastTxId() != HdfsConstants.InvalidTxid && editLogLedgerMetadata .GetLastTxId() < fromTxId) { // exclude already read closed edits, but include inprogress edits // as this will be handled in caller continue; } ledgers.AddItem(editLogLedgerMetadata); } catch (KeeperException.NoNodeException) { Log.Warn("ZNode: " + legderMetadataPath + " might have finalized and deleted." + " So ignoring NoNodeException."); } } } catch (KeeperException e) { throw new IOException("Exception reading ledger list from zk", e); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Interrupted getting list of ledgers from zk", ie); } ledgers.Sort(EditLogLedgerMetadata.Comparator); return(ledgers); }
/// <summary>Finalize a log segment.</summary> /// <remarks> /// Finalize a log segment. If the journal manager is currently /// writing to a ledger, ensure that this is the ledger of the log segment /// being finalized. /// Otherwise this is the recovery case. In the recovery case, ensure that /// the firstTxId of the ledger matches firstTxId for the segment we are /// trying to finalize. /// </remarks> /// <exception cref="System.IO.IOException"/> public override void FinalizeLogSegment(long firstTxId, long lastTxId) { CheckEnv(); string inprogressPath = InprogressZNode(firstTxId); try { Stat inprogressStat = zkc.Exists(inprogressPath, false); if (inprogressStat == null) { throw new IOException("Inprogress znode " + inprogressPath + " doesn't exist"); } EditLogLedgerMetadata l = EditLogLedgerMetadata.Read(zkc, inprogressPath); if (currentLedger != null) { // normal, non-recovery case if (l.GetLedgerId() == currentLedger.GetId()) { try { currentLedger.Close(); } catch (BKException bke) { Log.Error("Error closing current ledger", bke); } currentLedger = null; } else { throw new IOException("Active ledger has different ID to inprogress. " + l.GetLedgerId () + " found, " + currentLedger.GetId() + " expected"); } } if (l.GetFirstTxId() != firstTxId) { throw new IOException("Transaction id not as expected, " + l.GetFirstTxId() + " found, " + firstTxId + " expected"); } l.FinalizeLedger(lastTxId); string finalisedPath = FinalizedLedgerZNode(firstTxId, lastTxId); try { l.Write(zkc, finalisedPath); } catch (KeeperException.NodeExistsException) { if (!l.Verify(zkc, finalisedPath)) { throw new IOException("Node " + finalisedPath + " already exists" + " but data doesn't match" ); } } maxTxId.Store(lastTxId); zkc.Delete(inprogressPath, inprogressStat.GetVersion()); string inprogressPathFromCI = ci.Read(); if (inprogressPath.Equals(inprogressPathFromCI)) { ci.Clear(); } } catch (KeeperException e) { throw new IOException("Error finalising ledger", e); } catch (Exception ie) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Error finalising ledger", ie); } }