/// <exception cref="System.IO.IOException"/>
        public virtual void SelectInputStreams(ICollection <EditLogInputStream> streams, long
                                               fromTxnId, bool inProgressOk)
        {
            QuorumCall <AsyncLogger, RemoteEditLogManifest> q = loggers.GetEditLogManifest(fromTxnId
                                                                                           , inProgressOk);
            IDictionary <AsyncLogger, RemoteEditLogManifest> resps = loggers.WaitForWriteQuorum
                                                                         (q, selectInputStreamsTimeoutMs, "selectInputStreams");

            Log.Debug("selectInputStream manifests:\n" + Joiner.On("\n").WithKeyValueSeparator
                          (": ").Join(resps));
            PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
                                                                               >(64, JournalSet.EditLogInputStreamComparator);

            foreach (KeyValuePair <AsyncLogger, RemoteEditLogManifest> e in resps)
            {
                AsyncLogger           logger   = e.Key;
                RemoteEditLogManifest manifest = e.Value;
                foreach (RemoteEditLog remoteLog in manifest.GetLogs())
                {
                    Uri url = logger.BuildURLToFetchLogs(remoteLog.GetStartTxId());
                    EditLogInputStream elis = EditLogFileInputStream.FromUrl(connectionFactory, url,
                                                                             remoteLog.GetStartTxId(), remoteLog.GetEndTxId(), remoteLog.IsInProgress());
                    allStreams.AddItem(elis);
                }
            }
            JournalSet.ChainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
        }
示例#2
0
            public virtual AsyncLogger CreateLogger(Configuration conf, NamespaceInfo nsInfo,
                                                    string journalId, IPEndPoint addr)
            {
                AsyncLogger ch = IPCLoggerChannel.Factory.CreateLogger(conf, nsInfo, journalId, addr
                                                                       );
                AsyncLogger spy = Org.Mockito.Mockito.Spy(ch);

                Org.Mockito.Mockito.DoAnswer(new TestEpochsAreUnique.SometimesFaulty <long>(this,
                                                                                            0.10f)).When(spy).GetJournalState();
                Org.Mockito.Mockito.DoAnswer(new TestEpochsAreUnique.SometimesFaulty <Void>(this,
                                                                                            0.40f)).When(spy).NewEpoch(Org.Mockito.Mockito.AnyLong());
                return(spy);
            }
示例#3
0
 /// <summary>
 /// Append an HTML-formatted status readout on the current
 /// state of the underlying loggers.
 /// </summary>
 /// <param name="sb">the StringBuilder to append to</param>
 internal virtual void AppendReport(StringBuilder sb)
 {
     for (int i = 0; i < len; ++i)
     {
         AsyncLogger l = loggers[i];
         if (i != 0)
         {
             sb.Append(", ");
         }
         sb.Append(l).Append(" (");
         l.AppendReport(sb);
         sb.Append(")");
     }
 }
 private void FailLoggerAtTxn(AsyncLogger spy, long txid)
 {
     TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spy
                                                                                     ).SendEdits(Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.Eq(txid), Org.Mockito.Mockito
                                                                                                 .Eq(1), Org.Mockito.Mockito.Any <byte[]>());
 }
        /// <summary>Run recovery/synchronization for a specific segment.</summary>
        /// <remarks>
        /// Run recovery/synchronization for a specific segment.
        /// Postconditions:
        /// <ul>
        /// <li>This segment will be finalized on a majority
        /// of nodes.</li>
        /// <li>All nodes which contain the finalized segment will
        /// agree on the length.</li>
        /// </ul>
        /// </remarks>
        /// <param name="segmentTxId">the starting txid of the segment</param>
        /// <exception cref="System.IO.IOException"/>
        private void RecoverUnclosedSegment(long segmentTxId)
        {
            Preconditions.CheckArgument(segmentTxId > 0);
            Log.Info("Beginning recovery of unclosed segment starting at txid " + segmentTxId
                     );
            // Step 1. Prepare recovery
            QuorumCall <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepare
                = loggers.PrepareRecovery(segmentTxId);
            IDictionary <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepareResponses
                = loggers.WaitForWriteQuorum(prepare, prepareRecoveryTimeoutMs, "prepareRecovery("
                                             + segmentTxId + ")");

            Log.Info("Recovery prepare phase complete. Responses:\n" + QuorumCall.MapToString
                         (prepareResponses));
            // Determine the logger who either:
            // a) Has already accepted a previous proposal that's higher than any
            //    other
            //
            //  OR, if no such logger exists:
            //
            // b) Has the longest log starting at this transaction ID
            // TODO: we should collect any "ties" and pass the URL for all of them
            // when syncing, so we can tolerate failure during recovery better.
            KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> bestEntry
                = Sharpen.Collections.Max(prepareResponses, SegmentRecoveryComparator.Instance);
            AsyncLogger bestLogger = bestEntry.Key;

            QJournalProtocolProtos.PrepareRecoveryResponseProto bestResponse = bestEntry.Value;
            // Log the above decision, check invariants.
            if (bestResponse.HasAcceptedInEpoch())
            {
                Log.Info("Using already-accepted recovery for segment " + "starting at txid " + segmentTxId
                         + ": " + bestEntry);
            }
            else
            {
                if (bestResponse.HasSegmentState())
                {
                    Log.Info("Using longest log: " + bestEntry);
                }
                else
                {
                    // None of the responses to prepareRecovery() had a segment at the given
                    // txid. This can happen for example in the following situation:
                    // - 3 JNs: JN1, JN2, JN3
                    // - writer starts segment 101 on JN1, then crashes before
                    //   writing to JN2 and JN3
                    // - during newEpoch(), we saw the segment on JN1 and decide to
                    //   recover segment 101
                    // - before prepare(), JN1 crashes, and we only talk to JN2 and JN3,
                    //   neither of which has any entry for this log.
                    // In this case, it is allowed to do nothing for recovery, since the
                    // segment wasn't started on a quorum of nodes.
                    // Sanity check: we should only get here if none of the responses had
                    // a log. This should be a postcondition of the recovery comparator,
                    // but a bug in the comparator might cause us to get here.
                    foreach (QJournalProtocolProtos.PrepareRecoveryResponseProto resp in prepareResponses
                             .Values)
                    {
                        System.Diagnostics.Debug.Assert(!resp.HasSegmentState(), "One of the loggers had a response, but no best logger "
                                                        + "was found.");
                    }
                    Log.Info("None of the responders had a log to recover: " + QuorumCall.MapToString
                                 (prepareResponses));
                    return;
                }
            }
            QJournalProtocolProtos.SegmentStateProto logToSync = bestResponse.GetSegmentState
                                                                     ();
            System.Diagnostics.Debug.Assert(segmentTxId == logToSync.GetStartTxId());
            // Sanity check: none of the loggers should be aware of a higher
            // txid than the txid we intend to truncate to
            foreach (KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
                                   > e in prepareResponses)
            {
                AsyncLogger logger = e.Key;
                QJournalProtocolProtos.PrepareRecoveryResponseProto resp = e.Value;
                if (resp.HasLastCommittedTxId() && resp.GetLastCommittedTxId() > logToSync.GetEndTxId
                        ())
                {
                    throw new Exception("Decided to synchronize log to " + logToSync + " but logger "
                                        + logger + " had seen txid " + resp.GetLastCommittedTxId() + " committed");
                }
            }
            Uri syncFromUrl = bestLogger.BuildURLToFetchLogs(segmentTxId);
            QuorumCall <AsyncLogger, Void> accept = loggers.AcceptRecovery(logToSync, syncFromUrl
                                                                           );

            loggers.WaitForWriteQuorum(accept, acceptRecoveryTimeoutMs, "acceptRecovery(" + TextFormat
                                       .ShortDebugString(logToSync) + ")");
            // If one of the loggers above missed the synchronization step above, but
            // we send a finalize() here, that's OK. It validates the log before
            // finalizing. Hence, even if it is not "in sync", it won't incorrectly
            // finalize.
            QuorumCall <AsyncLogger, Void> finalize = loggers.FinalizeLogSegment(logToSync.GetStartTxId
                                                                                     (), logToSync.GetEndTxId());

            loggers.WaitForWriteQuorum(finalize, finalizeSegmentTimeoutMs, string.Format("finalizeLogSegment(%s-%s)"
                                                                                         , logToSync.GetStartTxId(), logToSync.GetEndTxId()));
        }
示例#6
0
 /// <summary>
 /// Inject a failure at the given IPC number, such that the JN never
 /// receives the RPC.
 /// </summary>
 /// <remarks>
 /// Inject a failure at the given IPC number, such that the JN never
 /// receives the RPC. The client side sees an IOException. Future
 /// IPCs after this number will be received as usual.
 /// </remarks>
 private void FailIpcNumber(AsyncLogger logger, int idx)
 {
     ((TestQJMWithFaults.InvocationCountingChannel)logger).FailIpcNumber(idx);
 }