/// <exception cref="System.IO.IOException"/>
public override void RecoverUnfinalizedSegments()
{
Preconditions.CheckState(!isActiveWriter, "already active writer");
Log.Info("Starting recovery process for unclosed journal segments...");
IDictionary <AsyncLogger, QJournalProtocolProtos.NewEpochResponseProto> resps = CreateNewUniqueEpoch
();
Log.Info("Successfully started new epoch " + loggers.GetEpoch());
if (Log.IsDebugEnabled())
{
Log.Debug("newEpoch(" + loggers.GetEpoch() + ") responses:\n" + QuorumCall.MapToString
(resps));
}
long mostRecentSegmentTxId = long.MinValue;
foreach (QJournalProtocolProtos.NewEpochResponseProto r in resps.Values)
{
if (r.HasLastSegmentTxId())
{
mostRecentSegmentTxId = Math.Max(mostRecentSegmentTxId, r.GetLastSegmentTxId());
}
}
// On a completely fresh system, none of the journals have any
// segments, so there's nothing to recover.
if (mostRecentSegmentTxId != long.MinValue)
{
RecoverUnclosedSegment(mostRecentSegmentTxId);
}
isActiveWriter = true;
}
/// <summary>Run recovery/synchronization for a specific segment.</summary>
/// <remarks>
/// Run recovery/synchronization for a specific segment.
/// Postconditions:
/// <ul>
/// <li>This segment will be finalized on a majority
/// of nodes.</li>
/// <li>All nodes which contain the finalized segment will
/// agree on the length.</li>
/// </ul>
/// </remarks>
/// <param name="segmentTxId">the starting txid of the segment</param>
/// <exception cref="System.IO.IOException"/>
private void RecoverUnclosedSegment(long segmentTxId)
{
Preconditions.CheckArgument(segmentTxId > 0);
Log.Info("Beginning recovery of unclosed segment starting at txid " + segmentTxId
);
// Step 1. Prepare recovery
QuorumCall <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepare
= loggers.PrepareRecovery(segmentTxId);
IDictionary <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepareResponses
= loggers.WaitForWriteQuorum(prepare, prepareRecoveryTimeoutMs, "prepareRecovery("
+ segmentTxId + ")");
Log.Info("Recovery prepare phase complete. Responses:\n" + QuorumCall.MapToString
(prepareResponses));
// Determine the logger who either:
// a) Has already accepted a previous proposal that's higher than any
// other
//
// OR, if no such logger exists:
//
// b) Has the longest log starting at this transaction ID
// TODO: we should collect any "ties" and pass the URL for all of them
// when syncing, so we can tolerate failure during recovery better.
KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> bestEntry
= Sharpen.Collections.Max(prepareResponses, SegmentRecoveryComparator.Instance);
AsyncLogger bestLogger = bestEntry.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto bestResponse = bestEntry.Value;
// Log the above decision, check invariants.
if (bestResponse.HasAcceptedInEpoch())
{
Log.Info("Using already-accepted recovery for segment " + "starting at txid " + segmentTxId
+ ": " + bestEntry);
}
else
{
if (bestResponse.HasSegmentState())
{
Log.Info("Using longest log: " + bestEntry);
}
else
{
// None of the responses to prepareRecovery() had a segment at the given
// txid. This can happen for example in the following situation:
// - 3 JNs: JN1, JN2, JN3
// - writer starts segment 101 on JN1, then crashes before
// writing to JN2 and JN3
// - during newEpoch(), we saw the segment on JN1 and decide to
// recover segment 101
// - before prepare(), JN1 crashes, and we only talk to JN2 and JN3,
// neither of which has any entry for this log.
// In this case, it is allowed to do nothing for recovery, since the
// segment wasn't started on a quorum of nodes.
// Sanity check: we should only get here if none of the responses had
// a log. This should be a postcondition of the recovery comparator,
// but a bug in the comparator might cause us to get here.
foreach (QJournalProtocolProtos.PrepareRecoveryResponseProto resp in prepareResponses
.Values)
{
System.Diagnostics.Debug.Assert(!resp.HasSegmentState(), "One of the loggers had a response, but no best logger "
+ "was found.");
}
Log.Info("None of the responders had a log to recover: " + QuorumCall.MapToString
(prepareResponses));
return;
}
}
QJournalProtocolProtos.SegmentStateProto logToSync = bestResponse.GetSegmentState
();
System.Diagnostics.Debug.Assert(segmentTxId == logToSync.GetStartTxId());
// Sanity check: none of the loggers should be aware of a higher
// txid than the txid we intend to truncate to
foreach (KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
> e in prepareResponses)
{
AsyncLogger logger = e.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto resp = e.Value;
if (resp.HasLastCommittedTxId() && resp.GetLastCommittedTxId() > logToSync.GetEndTxId
())
{
throw new Exception("Decided to synchronize log to " + logToSync + " but logger "
+ logger + " had seen txid " + resp.GetLastCommittedTxId() + " committed");
}
}
Uri syncFromUrl = bestLogger.BuildURLToFetchLogs(segmentTxId);
QuorumCall <AsyncLogger, Void> accept = loggers.AcceptRecovery(logToSync, syncFromUrl
);
loggers.WaitForWriteQuorum(accept, acceptRecoveryTimeoutMs, "acceptRecovery(" + TextFormat
.ShortDebugString(logToSync) + ")");
// If one of the loggers above missed the synchronization step above, but
// we send a finalize() here, that's OK. It validates the log before
// finalizing. Hence, even if it is not "in sync", it won't incorrectly
// finalize.
QuorumCall <AsyncLogger, Void> finalize = loggers.FinalizeLogSegment(logToSync.GetStartTxId
(), logToSync.GetEndTxId());
loggers.WaitForWriteQuorum(finalize, finalizeSegmentTimeoutMs, string.Format("finalizeLogSegment(%s-%s)"
, logToSync.GetStartTxId(), logToSync.GetEndTxId()));
}