public virtual void TestRecoverAfterIncompleteRecovery()
{
SetupLoggers345();
// Shut down the logger that has length = 5
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
// Allow no logger to finalize
foreach (AsyncLogger spy in spies)
{
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spy).
FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(4L));
}
try
{
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed recovery since no finalization occurred"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("injected", ioe);
}
// Now bring back the logger that had 5, and run recovery again.
// We should recover to 4, even though there's a longer log.
cluster.GetJournalNode(0).StopAndJoin(0);
cluster.RestartJournalNode(2);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 4);
}
public virtual void TestMissFinalizeAndNextStart()
{
// Logger 0: miss finalize(1-3) and start(4)
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(3L));
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spies
[0]).StartLogSegment(Org.Mockito.Mockito.Eq(4L), Org.Mockito.Mockito.Eq(NameNodeLayoutVersion
.CurrentLayoutVersion));
// Logger 1: fail at txn id 4
FailLoggerAtTxn(spies[1], 4L);
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
EditLogOutputStream stm = qjm.StartLogSegment(4, NameNodeLayoutVersion.CurrentLayoutVersion
);
try
{
QJMTestUtil.WriteTxns(stm, 4, 1);
NUnit.Framework.Assert.Fail("Did not fail to write");
}
catch (QuorumException qe)
{
// Should fail, because logger 1 had an injected fault and
// logger 0 should detect writer out of sync
GenericTestUtils.AssertExceptionContains("Writer out of sync", qe);
}
finally
{
stm.Abort();
qjm.Close();
}
// State:
// Logger 0: 1-3 in-progress (since it missed finalize)
// Logger 1: 1-3 finalized
// Logger 2: 1-3 finalized, 4 in-progress with one txn
// Shut down logger 2 so it doesn't participate in recovery
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
long recovered = QJMTestUtil.RecoverAndReturnLastTxn(qjm);
NUnit.Framework.Assert.AreEqual(3L, recovered);
}
private Stubber InjectIOE()
{
return(TestQuorumJournalManagerUnit.FutureThrows(new IOException("Injected")));
}
/// <summary>
/// Set up the following tricky edge case state which is used by
/// multiple tests:
/// Initial writer:
/// - Writing to 3 JNs: JN0, JN1, JN2:
/// - A log segment with txnid 1 through 100 succeeds.
/// </summary>
/// <remarks>
/// Set up the following tricky edge case state which is used by
/// multiple tests:
/// Initial writer:
/// - Writing to 3 JNs: JN0, JN1, JN2:
/// - A log segment with txnid 1 through 100 succeeds.
/// - The first transaction in the next segment only goes to JN0
/// before the writer crashes (eg it is partitioned)
/// Recovery by another writer:
/// - The new NN starts recovery and talks to all three. Thus, it sees
/// that the newest log segment which needs recovery is 101.
/// - It sends the prepareRecovery(101) call, and decides that the
/// recovery length for 101 is only the 1 transaction.
/// - It sends acceptRecovery(101-101) to only JN0, before crashing
/// This yields the following state:
/// - JN0: 1-100 finalized, 101_inprogress, accepted recovery: 101-101
/// - JN1: 1-100 finalized, 101_inprogress.empty
/// - JN2: 1-100 finalized, 101_inprogress.empty
/// (the .empty files got moved aside during recovery)
/// </remarks>
/// <exception cref="System.Exception"></exception>
private void SetupEdgeCaseOneJnHasSegmentWithAcceptedRecovery()
{
// Log segment with txns 1-100 succeeds
QJMTestUtil.WriteSegment(cluster, qjm, 1, 100, true);
// startLogSegment only makes it to one of the three nodes
FailLoggerAtTxn(spies[1], 101);
FailLoggerAtTxn(spies[2], 101);
try
{
QJMTestUtil.WriteSegment(cluster, qjm, 101, 1, true);
NUnit.Framework.Assert.Fail("Should have failed");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
finally
{
qjm.Close();
}
// Recovery 1:
// make acceptRecovery() only make it to the node which has txid 101
// this should fail because only 1/3 accepted the recovery
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies
[1]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto
>(), Org.Mockito.Mockito.Any <Uri>());
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spies
[2]).AcceptRecovery(Org.Mockito.Mockito.Any <QJournalProtocolProtos.SegmentStateProto
>(), Org.Mockito.Mockito.Any <Uri>());
try
{
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed to recover");
}
catch (QuorumException qe)
{
GenericTestUtils.AssertExceptionContains("mock failure", qe);
}
finally
{
qjm.Close();
}
// Check that we have entered the expected state as described in the
// method javadoc.
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101));
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101) + ".empty");
GenericTestUtils.AssertGlobEquals(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "edits_.*"
, NNStorage.GetFinalizedEditsFileName(1, 100), NNStorage.GetInProgressEditsFileName
(101) + ".empty");
FilePath paxos0 = new FilePath(cluster.GetCurrentDir(0, QJMTestUtil.Jid), "paxos"
);
FilePath paxos1 = new FilePath(cluster.GetCurrentDir(1, QJMTestUtil.Jid), "paxos"
);
FilePath paxos2 = new FilePath(cluster.GetCurrentDir(2, QJMTestUtil.Jid), "paxos"
);
GenericTestUtils.AssertGlobEquals(paxos0, ".*", "101");
GenericTestUtils.AssertGlobEquals(paxos1, ".*");
GenericTestUtils.AssertGlobEquals(paxos2, ".*");
}
private void FailLoggerAtTxn(AsyncLogger spy, long txid)
{
TestQuorumJournalManagerUnit.FutureThrows(new IOException("mock failure")).When(spy
).SendEdits(Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.Eq(txid), Org.Mockito.Mockito
.Eq(1), Org.Mockito.Mockito.Any <byte[]>());
}