/// <exception cref="System.IO.IOException"/>
public virtual void SelectInputStreams(ICollection <EditLogInputStream> streams, long
fromTxnId, bool inProgressOk)
{
QuorumCall <AsyncLogger, RemoteEditLogManifest> q = loggers.GetEditLogManifest(fromTxnId
, inProgressOk);
IDictionary <AsyncLogger, RemoteEditLogManifest> resps = loggers.WaitForWriteQuorum
(q, selectInputStreamsTimeoutMs, "selectInputStreams");
Log.Debug("selectInputStream manifests:\n" + Joiner.On("\n").WithKeyValueSeparator
(": ").Join(resps));
PriorityQueue <EditLogInputStream> allStreams = new PriorityQueue <EditLogInputStream
>(64, JournalSet.EditLogInputStreamComparator);
foreach (KeyValuePair <AsyncLogger, RemoteEditLogManifest> e in resps)
{
AsyncLogger logger = e.Key;
RemoteEditLogManifest manifest = e.Value;
foreach (RemoteEditLog remoteLog in manifest.GetLogs())
{
Uri url = logger.BuildURLToFetchLogs(remoteLog.GetStartTxId());
EditLogInputStream elis = EditLogFileInputStream.FromUrl(connectionFactory, url,
remoteLog.GetStartTxId(), remoteLog.GetEndTxId(), remoteLog.IsInProgress());
allStreams.AddItem(elis);
}
}
JournalSet.ChainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
public virtual AsyncLogger CreateLogger(Configuration conf, NamespaceInfo nsInfo,
string journalId, IPEndPoint addr)
{
AsyncLogger ch = IPCLoggerChannel.Factory.CreateLogger(conf, nsInfo, journalId, addr
);
AsyncLogger spy = Org.Mockito.Mockito.Spy(ch);
Org.Mockito.Mockito.DoAnswer(new TestEpochsAreUnique.SometimesFaulty <long>(this,
0.10f)).When(spy).GetJournalState();
Org.Mockito.Mockito.DoAnswer(new TestEpochsAreUnique.SometimesFaulty <Void>(this,
0.40f)).When(spy).NewEpoch(Org.Mockito.Mockito.AnyLong());
return(spy);
}
/// <summary>
/// Append an HTML-formatted status readout on the current
/// state of the underlying loggers.
/// </summary>
/// <param name="sb">the StringBuilder to append to</param>
internal virtual void AppendReport(StringBuilder sb)
{
for (int i = 0; i < len; ++i)
{
AsyncLogger l = loggers[i];
if (i != 0)
{
sb.Append(", ");
}
sb.Append(l).Append(" (");
l.AppendReport(sb);
sb.Append(")");
}
}
private void FailLoggerAtTxn(AsyncLogger spy, long txid)
{
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spy
).SendEdits(Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.Eq(txid), Org.Mockito.Mockito
.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
}
/// <summary>Run recovery/synchronization for a specific segment.</summary>
/// <remarks>
/// Run recovery/synchronization for a specific segment.
/// Postconditions:
/// <ul>
/// <li>This segment will be finalized on a majority
/// of nodes.</li>
/// <li>All nodes which contain the finalized segment will
/// agree on the length.</li>
/// </ul>
/// </remarks>
/// <param name="segmentTxId">the starting txid of the segment</param>
/// <exception cref="System.IO.IOException"/>
private void RecoverUnclosedSegment(long segmentTxId)
{
Preconditions.CheckArgument(segmentTxId > 0);
Log.Info("Beginning recovery of unclosed segment starting at txid " + segmentTxId
);
// Step 1. Prepare recovery
QuorumCall <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepare
= loggers.PrepareRecovery(segmentTxId);
IDictionary <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> prepareResponses
= loggers.WaitForWriteQuorum(prepare, prepareRecoveryTimeoutMs, "prepareRecovery("
+ segmentTxId + ")");
Log.Info("Recovery prepare phase complete. Responses:\n" + QuorumCall.MapToString
(prepareResponses));
// Determine the logger who either:
// a) Has already accepted a previous proposal that's higher than any
// other
//
// OR, if no such logger exists:
//
// b) Has the longest log starting at this transaction ID
// TODO: we should collect any "ties" and pass the URL for all of them
// when syncing, so we can tolerate failure during recovery better.
KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto> bestEntry
= Sharpen.Collections.Max(prepareResponses, SegmentRecoveryComparator.Instance);
AsyncLogger bestLogger = bestEntry.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto bestResponse = bestEntry.Value;
// Log the above decision, check invariants.
if (bestResponse.HasAcceptedInEpoch())
{
Log.Info("Using already-accepted recovery for segment " + "starting at txid " + segmentTxId
+ ": " + bestEntry);
}
else
{
if (bestResponse.HasSegmentState())
{
Log.Info("Using longest log: " + bestEntry);
}
else
{
// None of the responses to prepareRecovery() had a segment at the given
// txid. This can happen for example in the following situation:
// - 3 JNs: JN1, JN2, JN3
// - writer starts segment 101 on JN1, then crashes before
// writing to JN2 and JN3
// - during newEpoch(), we saw the segment on JN1 and decide to
// recover segment 101
// - before prepare(), JN1 crashes, and we only talk to JN2 and JN3,
// neither of which has any entry for this log.
// In this case, it is allowed to do nothing for recovery, since the
// segment wasn't started on a quorum of nodes.
// Sanity check: we should only get here if none of the responses had
// a log. This should be a postcondition of the recovery comparator,
// but a bug in the comparator might cause us to get here.
foreach (QJournalProtocolProtos.PrepareRecoveryResponseProto resp in prepareResponses
.Values)
{
System.Diagnostics.Debug.Assert(!resp.HasSegmentState(), "One of the loggers had a response, but no best logger "
+ "was found.");
}
Log.Info("None of the responders had a log to recover: " + QuorumCall.MapToString
(prepareResponses));
return;
}
}
QJournalProtocolProtos.SegmentStateProto logToSync = bestResponse.GetSegmentState
();
System.Diagnostics.Debug.Assert(segmentTxId == logToSync.GetStartTxId());
// Sanity check: none of the loggers should be aware of a higher
// txid than the txid we intend to truncate to
foreach (KeyValuePair <AsyncLogger, QJournalProtocolProtos.PrepareRecoveryResponseProto
> e in prepareResponses)
{
AsyncLogger logger = e.Key;
QJournalProtocolProtos.PrepareRecoveryResponseProto resp = e.Value;
if (resp.HasLastCommittedTxId() && resp.GetLastCommittedTxId() > logToSync.GetEndTxId
())
{
throw new Exception("Decided to synchronize log to " + logToSync + " but logger "
+ logger + " had seen txid " + resp.GetLastCommittedTxId() + " committed");
}
}
Uri syncFromUrl = bestLogger.BuildURLToFetchLogs(segmentTxId);
QuorumCall <AsyncLogger, Void> accept = loggers.AcceptRecovery(logToSync, syncFromUrl
);
loggers.WaitForWriteQuorum(accept, acceptRecoveryTimeoutMs, "acceptRecovery(" + TextFormat
.ShortDebugString(logToSync) + ")");
// If one of the loggers above missed the synchronization step above, but
// we send a finalize() here, that's OK. It validates the log before
// finalizing. Hence, even if it is not "in sync", it won't incorrectly
// finalize.
QuorumCall <AsyncLogger, Void> finalize = loggers.FinalizeLogSegment(logToSync.GetStartTxId
(), logToSync.GetEndTxId());
loggers.WaitForWriteQuorum(finalize, finalizeSegmentTimeoutMs, string.Format("finalizeLogSegment(%s-%s)"
, logToSync.GetStartTxId(), logToSync.GetEndTxId()));
}
/// <summary>
/// Inject a failure at the given IPC number, such that the JN never
/// receives the RPC.
/// </summary>
/// <remarks>
/// Inject a failure at the given IPC number, such that the JN never
/// receives the RPC. The client side sees an IOException. Future
/// IPCs after this number will be received as usual.
/// </remarks>
private void FailIpcNumber(AsyncLogger logger, int idx)
{
((TestQJMWithFaults.InvocationCountingChannel)logger).FailIpcNumber(idx);
}