public virtual void TestScanEditLog()
{
// use a future layout version
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion
- 1);
// in the segment we write garbage editlog, which can be scanned but
// cannot be decoded
int numTxns = 5;
byte[] ops = QJMTestUtil.CreateGabageTxns(1, 5);
journal.Journal(MakeRI(2), 1, 1, numTxns, ops);
// verify the in-progress editlog segment
QJournalProtocolProtos.SegmentStateProto segmentState = journal.GetSegmentInfo(1);
NUnit.Framework.Assert.IsTrue(segmentState.GetIsInProgress());
NUnit.Framework.Assert.AreEqual(numTxns, segmentState.GetEndTxId());
NUnit.Framework.Assert.AreEqual(1, segmentState.GetStartTxId());
// finalize the segment and verify it again
journal.FinalizeLogSegment(MakeRI(3), 1, numTxns);
segmentState = journal.GetSegmentInfo(1);
NUnit.Framework.Assert.IsFalse(segmentState.GetIsInProgress());
NUnit.Framework.Assert.AreEqual(numTxns, segmentState.GetEndTxId());
NUnit.Framework.Assert.AreEqual(1, segmentState.GetStartTxId());
}
public virtual int Compare(KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
> a, KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
> b)
{
QJournalProtocolProtos.PrepareRecoveryResponseProto r1 = a.Value;
QJournalProtocolProtos.PrepareRecoveryResponseProto r2 = b.Value;
// A response that has data for a segment is always better than one
// that doesn't.
if (r1.HasSegmentState() != r2.HasSegmentState())
{
return(Booleans.Compare(r1.HasSegmentState(), r2.HasSegmentState()));
}
if (!r1.HasSegmentState())
{
// Neither has a segment, so neither can be used for recover.
// Call them equal.
return(0);
}
// They both have a segment.
QJournalProtocolProtos.SegmentStateProto r1Seg = r1.GetSegmentState();
QJournalProtocolProtos.SegmentStateProto r2Seg = r2.GetSegmentState();
Preconditions.CheckArgument(r1Seg.GetStartTxId() == r2Seg.GetStartTxId(), "Should only be called with responses for corresponding segments: "
+ "%s and %s do not have the same start txid.", r1, r2);
// If one is in-progress but the other is finalized,
// the finalized one is greater.
if (r1Seg.GetIsInProgress() != r2Seg.GetIsInProgress())
{
return(Booleans.Compare(!r1Seg.GetIsInProgress(), !r2Seg.GetIsInProgress()));
}
if (!r1Seg.GetIsInProgress())
{
// If both are finalized, they should match lengths
if (r1Seg.GetEndTxId() != r2Seg.GetEndTxId())
{
throw new Exception("finalized segs with different lengths: " + r1 + ", " + r2);
}
return(0);
}
// Both are in-progress.
long r1SeenEpoch = Math.Max(r1.GetAcceptedInEpoch(), r1.GetLastWriterEpoch());
long r2SeenEpoch = Math.Max(r2.GetAcceptedInEpoch(), r2.GetLastWriterEpoch());
return(ComparisonChain.Start().Compare(r1SeenEpoch, r2SeenEpoch).Compare(r1.GetSegmentState
().GetEndTxId(), r2.GetSegmentState().GetEndTxId()).Result());
}
/// <summary>Run recovery/synchronization for a specific segment.</summary>
/// <remarks>
/// Run recovery/synchronization for a specific segment.
/// Postconditions:
/// <ul>
/// <li>This segment will be finalized on a majority
/// of nodes.</li>
/// <li>All nodes which contain the finalized segment will
/// agree on the length.</li>
/// </ul>
/// </remarks>
/// <param name="segmentTxId">the starting txid of the segment</param>
/// <exception cref="System.IO.IOException"/>
private void RecoverUnclosedSegment(long segmentTxId)
{
Preconditions.CheckArgument(segmentTxId > 0);
Log.Info("Beginning recovery of unclosed segment starting at txid " + segmentTxId
);
// Step 1. Prepare recovery
QuorumCall <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepare
= loggers.PrepareRecovery(segmentTxId);
IDictionary <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepareResponses
= loggers.WaitForWriteQuorum(prepare, prepareRecoveryTimeoutMs, "prepareRecovery("
+ segmentTxId + ")");
Log.Info("Recovery prepare phase complete. Responses:\n" + QuorumCall.MapToString
(prepareResponses));
// Determine the logger who either:
// a) Has already accepted a previous proposal that's higher than any
// other
//
// OR, if no such logger exists:
//
// b) Has the longest log starting at this transaction ID
// TODO: we should collect any "ties" and pass the URL for all of them
// when syncing, so we can tolerate failure during recovery better.
KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> bestEntry
= Sharpen.Collections.Max(prepareResponses, SegmentRecoveryComparator.Instance);
AsyncLogger bestLogger = bestEntry.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto bestResponse = bestEntry.Value;
// Log the above decision, check invariants.
if (bestResponse.HasAcceptedInEpoch())
{
Log.Info("Using already-accepted recovery for segment " + "starting at txid " + segmentTxId
+ ": " + bestEntry);
}
else
{
if (bestResponse.HasSegmentState())
{
Log.Info("Using longest log: " + bestEntry);
}
else
{
// None of the responses to prepareRecovery() had a segment at the given
// txid. This can happen for example in the following situation:
// - 3 JNs: JN1, JN2, JN3
// - writer starts segment 101 on JN1, then crashes before
// writing to JN2 and JN3
// - during newEpoch(), we saw the segment on JN1 and decide to
// recover segment 101
// - before prepare(), JN1 crashes, and we only talk to JN2 and JN3,
// neither of which has any entry for this log.
// In this case, it is allowed to do nothing for recovery, since the
// segment wasn't started on a quorum of nodes.
// Sanity check: we should only get here if none of the responses had
// a log. This should be a postcondition of the recovery comparator,
// but a bug in the comparator might cause us to get here.
foreach (QJournalProtocolProtos.PrepareRecoveryResponseProto resp in prepareResponses
.Values)
{
System.Diagnostics.Debug.Assert(!resp.HasSegmentState(), "One of the loggers had a response, but no best logger "
+ "was found.");
}
Log.Info("None of the responders had a log to recover: " + QuorumCall.MapToString
(prepareResponses));
return;
}
}
QJournalProtocolProtos.SegmentStateProto logToSync = bestResponse.GetSegmentState
();
System.Diagnostics.Debug.Assert(segmentTxId == logToSync.GetStartTxId());
// Sanity check: none of the loggers should be aware of a higher
// txid than the txid we intend to truncate to
foreach (KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
> e in prepareResponses)
{
AsyncLogger logger = e.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto resp = e.Value;
if (resp.HasLastCommittedTxId() && resp.GetLastCommittedTxId() > logToSync.GetEndTxId
())
{
throw new Exception("Decided to synchronize log to " + logToSync + " but logger "
+ logger + " had seen txid " + resp.GetLastCommittedTxId() + " committed");
}
}
Uri syncFromUrl = bestLogger.BuildURLToFetchLogs(segmentTxId);
QuorumCall <AsyncLogger, Void> accept = loggers.AcceptRecovery(logToSync, syncFromUrl
);
loggers.WaitForWriteQuorum(accept, acceptRecoveryTimeoutMs, "acceptRecovery(" + TextFormat
.ShortDebugString(logToSync) + ")");
// If one of the loggers above missed the synchronization step above, but
// we send a finalize() here, that's OK. It validates the log before
// finalizing. Hence, even if it is not "in sync", it won't incorrectly
// finalize.
QuorumCall <AsyncLogger, Void> finalize = loggers.FinalizeLogSegment(logToSync.GetStartTxId
(), logToSync.GetEndTxId());
loggers.WaitForWriteQuorum(finalize, finalizeSegmentTimeoutMs, string.Format("finalizeLogSegment(%s-%s)"
, logToSync.GetStartTxId(), logToSync.GetEndTxId()));
}