Example #1
0
		public virtual void TestScanEditLog()
		{
			// use a future layout version
			journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion 
				- 1);
			// in the segment we write garbage editlog, which can be scanned but
			// cannot be decoded
			int numTxns = 5;
			byte[] ops = QJMTestUtil.CreateGabageTxns(1, 5);
			journal.Journal(MakeRI(2), 1, 1, numTxns, ops);
			// verify the in-progress editlog segment
			QJournalProtocolProtos.SegmentStateProto segmentState = journal.GetSegmentInfo(1);
			NUnit.Framework.Assert.IsTrue(segmentState.GetIsInProgress());
			NUnit.Framework.Assert.AreEqual(numTxns, segmentState.GetEndTxId());
			NUnit.Framework.Assert.AreEqual(1, segmentState.GetStartTxId());
			// finalize the segment and verify it again
			journal.FinalizeLogSegment(MakeRI(3), 1, numTxns);
			segmentState = journal.GetSegmentInfo(1);
			NUnit.Framework.Assert.IsFalse(segmentState.GetIsInProgress());
			NUnit.Framework.Assert.AreEqual(numTxns, segmentState.GetEndTxId());
			NUnit.Framework.Assert.AreEqual(1, segmentState.GetStartTxId());
		}
        public virtual int Compare(KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
                                                 > a, KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
                                                                    > b)
        {
            QJournalProtocolProtos.PrepareRecoveryResponseProto r1 = a.Value;
            QJournalProtocolProtos.PrepareRecoveryResponseProto r2 = b.Value;
            // A response that has data for a segment is always better than one
            // that doesn't.
            if (r1.HasSegmentState() != r2.HasSegmentState())
            {
                return(Booleans.Compare(r1.HasSegmentState(), r2.HasSegmentState()));
            }
            if (!r1.HasSegmentState())
            {
                // Neither has a segment, so neither can be used for recover.
                // Call them equal.
                return(0);
            }
            // They both have a segment.
            QJournalProtocolProtos.SegmentStateProto r1Seg = r1.GetSegmentState();
            QJournalProtocolProtos.SegmentStateProto r2Seg = r2.GetSegmentState();
            Preconditions.CheckArgument(r1Seg.GetStartTxId() == r2Seg.GetStartTxId(), "Should only be called with responses for corresponding segments: "
                                        + "%s and %s do not have the same start txid.", r1, r2);
            // If one is in-progress but the other is finalized,
            // the finalized one is greater.
            if (r1Seg.GetIsInProgress() != r2Seg.GetIsInProgress())
            {
                return(Booleans.Compare(!r1Seg.GetIsInProgress(), !r2Seg.GetIsInProgress()));
            }
            if (!r1Seg.GetIsInProgress())
            {
                // If both are finalized, they should match lengths
                if (r1Seg.GetEndTxId() != r2Seg.GetEndTxId())
                {
                    throw new Exception("finalized segs with different lengths: " + r1 + ", " + r2);
                }
                return(0);
            }
            // Both are in-progress.
            long r1SeenEpoch = Math.Max(r1.GetAcceptedInEpoch(), r1.GetLastWriterEpoch());
            long r2SeenEpoch = Math.Max(r2.GetAcceptedInEpoch(), r2.GetLastWriterEpoch());

            return(ComparisonChain.Start().Compare(r1SeenEpoch, r2SeenEpoch).Compare(r1.GetSegmentState
                                                                                         ().GetEndTxId(), r2.GetSegmentState().GetEndTxId()).Result());
        }
        /// <summary>Run recovery/synchronization for a specific segment.</summary>
        /// <remarks>
        /// Run recovery/synchronization for a specific segment.
        /// Postconditions:
        /// <ul>
        /// <li>This segment will be finalized on a majority
        /// of nodes.</li>
        /// <li>All nodes which contain the finalized segment will
        /// agree on the length.</li>
        /// </ul>
        /// </remarks>
        /// <param name="segmentTxId">the starting txid of the segment</param>
        /// <exception cref="System.IO.IOException"/>
        private void RecoverUnclosedSegment(long segmentTxId)
        {
            Preconditions.CheckArgument(segmentTxId > 0);
            Log.Info("Beginning recovery of unclosed segment starting at txid " + segmentTxId
                     );
            // Step 1. Prepare recovery
            QuorumCall <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepare
                = loggers.PrepareRecovery(segmentTxId);
            IDictionary <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepareResponses
                = loggers.WaitForWriteQuorum(prepare, prepareRecoveryTimeoutMs, "prepareRecovery("
                                             + segmentTxId + ")");

            Log.Info("Recovery prepare phase complete. Responses:\n" + QuorumCall.MapToString
                         (prepareResponses));
            // Determine the logger who either:
            // a) Has already accepted a previous proposal that's higher than any
            //    other
            //
            //  OR, if no such logger exists:
            //
            // b) Has the longest log starting at this transaction ID
            // TODO: we should collect any "ties" and pass the URL for all of them
            // when syncing, so we can tolerate failure during recovery better.
            KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> bestEntry
                = Sharpen.Collections.Max(prepareResponses, SegmentRecoveryComparator.Instance);
            AsyncLogger bestLogger = bestEntry.Key;

            QJournalProtocolProtos.PrepareRecoveryResponseProto bestResponse = bestEntry.Value;
            // Log the above decision, check invariants.
            if (bestResponse.HasAcceptedInEpoch())
            {
                Log.Info("Using already-accepted recovery for segment " + "starting at txid " + segmentTxId
                         + ": " + bestEntry);
            }
            else
            {
                if (bestResponse.HasSegmentState())
                {
                    Log.Info("Using longest log: " + bestEntry);
                }
                else
                {
                    // None of the responses to prepareRecovery() had a segment at the given
                    // txid. This can happen for example in the following situation:
                    // - 3 JNs: JN1, JN2, JN3
                    // - writer starts segment 101 on JN1, then crashes before
                    //   writing to JN2 and JN3
                    // - during newEpoch(), we saw the segment on JN1 and decide to
                    //   recover segment 101
                    // - before prepare(), JN1 crashes, and we only talk to JN2 and JN3,
                    //   neither of which has any entry for this log.
                    // In this case, it is allowed to do nothing for recovery, since the
                    // segment wasn't started on a quorum of nodes.
                    // Sanity check: we should only get here if none of the responses had
                    // a log. This should be a postcondition of the recovery comparator,
                    // but a bug in the comparator might cause us to get here.
                    foreach (QJournalProtocolProtos.PrepareRecoveryResponseProto resp in prepareResponses
                             .Values)
                    {
                        System.Diagnostics.Debug.Assert(!resp.HasSegmentState(), "One of the loggers had a response, but no best logger "
                                                        + "was found.");
                    }
                    Log.Info("None of the responders had a log to recover: " + QuorumCall.MapToString
                                 (prepareResponses));
                    return;
                }
            }
            QJournalProtocolProtos.SegmentStateProto logToSync = bestResponse.GetSegmentState
                                                                     ();
            System.Diagnostics.Debug.Assert(segmentTxId == logToSync.GetStartTxId());
            // Sanity check: none of the loggers should be aware of a higher
            // txid than the txid we intend to truncate to
            foreach (KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
                                   > e in prepareResponses)
            {
                AsyncLogger logger = e.Key;
                QJournalProtocolProtos.PrepareRecoveryResponseProto resp = e.Value;
                if (resp.HasLastCommittedTxId() && resp.GetLastCommittedTxId() > logToSync.GetEndTxId
                        ())
                {
                    throw new Exception("Decided to synchronize log to " + logToSync + " but logger "
                                        + logger + " had seen txid " + resp.GetLastCommittedTxId() + " committed");
                }
            }
            Uri syncFromUrl = bestLogger.BuildURLToFetchLogs(segmentTxId);
            QuorumCall <AsyncLogger, Void> accept = loggers.AcceptRecovery(logToSync, syncFromUrl
                                                                           );

            loggers.WaitForWriteQuorum(accept, acceptRecoveryTimeoutMs, "acceptRecovery(" + TextFormat
                                       .ShortDebugString(logToSync) + ")");
            // If one of the loggers above missed the synchronization step above, but
            // we send a finalize() here, that's OK. It validates the log before
            // finalizing. Hence, even if it is not "in sync", it won't incorrectly
            // finalize.
            QuorumCall <AsyncLogger, Void> finalize = loggers.FinalizeLogSegment(logToSync.GetStartTxId
                                                                                     (), logToSync.GetEndTxId());

            loggers.WaitForWriteQuorum(finalize, finalizeSegmentTimeoutMs, string.Format("finalizeLogSegment(%s-%s)"
                                                                                         , logToSync.GetStartTxId(), logToSync.GetEndTxId()));
        }