public virtual void TestTrySendErrorReportWhenNNThrowsIOException()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new _Answer_602(this)).When(mockNN1).ErrorReport(Org.Mockito.Mockito
.Any <DatanodeRegistration>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.
AnyString());
// Throw an IOException when this function is first called which will
// in turn add that errorReport back to the bpThreadQueue and let it
// process the next time.
string errorString = "Can't send invalid block " + FakeBlock;
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
Sharpen.Thread.Sleep(10000);
NUnit.Framework.Assert.IsTrue("Active namenode didn't add the report back to the queue "
+ "when errorReport threw IOException", secondCallTime != 0);
}
finally
{
bpos.Stop();
}
}
/// <summary>
/// Update the BPOS's view of which NN is active, based on a heartbeat
/// response from one of the actors.
/// </summary>
/// <param name="actor">the actor which received the heartbeat</param>
/// <param name="nnHaState">the HA-related heartbeat contents</param>
internal virtual void UpdateActorStatesFromHeartbeat(BPServiceActor actor, NNHAStatusHeartbeat
nnHaState)
{
WriteLock();
try
{
long txid = nnHaState.GetTxId();
bool nnClaimsActive = nnHaState.GetState() == HAServiceProtocol.HAServiceState.Active;
bool bposThinksActive = bpServiceToActive == actor;
bool isMoreRecentClaim = txid > lastActiveClaimTxId;
if (nnClaimsActive && !bposThinksActive)
{
Log.Info("Namenode " + actor + " trying to claim ACTIVE state with " + "txid=" +
txid);
if (!isMoreRecentClaim)
{
// Split-brain scenario - an NN is trying to claim active
// state when a different NN has already claimed it with a higher
// txid.
Log.Warn("NN " + actor + " tried to claim ACTIVE state at txid=" + txid + " but there was already a more recent claim at txid="
+ lastActiveClaimTxId);
return;
}
else
{
if (bpServiceToActive == null)
{
Log.Info("Acknowledging ACTIVE Namenode " + actor);
}
else
{
Log.Info("Namenode " + actor + " taking over ACTIVE state from " + bpServiceToActive
+ " at higher txid=" + txid);
}
bpServiceToActive = actor;
}
}
else
{
if (!nnClaimsActive && bposThinksActive)
{
Log.Info("Namenode " + actor + " relinquishing ACTIVE state with " + "txid=" + nnHaState
.GetTxId());
bpServiceToActive = null;
}
}
if (bpServiceToActive == actor)
{
System.Diagnostics.Debug.Assert(txid >= lastActiveClaimTxId);
lastActiveClaimTxId = txid;
}
}
finally
{
WriteUnlock();
}
}
/// <exception cref="System.IO.IOException"/>
private static void SetHeartbeatResponse(DatanodeCommand[] cmds)
{
NNHAStatusHeartbeat ha = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState
.Active, fsImage.GetLastAppliedOrWrittenTxId());
HeartbeatResponse response = new HeartbeatResponse(cmds, ha, null);
Org.Mockito.Mockito.DoReturn(response).When(spyNN).SendHeartbeat((DatanodeRegistration
)Matchers.Any(), (StorageReport[])Matchers.Any(), Matchers.AnyLong(), Matchers.AnyLong
(), Matchers.AnyInt(), Matchers.AnyInt(), Matchers.AnyInt(), (VolumeFailureSummary
)Matchers.Any());
}
/// <summary>Set up a mock NN with the bare minimum for a DN to register to it.</summary>
/// <exception cref="System.Exception"/>
private DatanodeProtocolClientSideTranslatorPB SetupNNMock(int nnIdx)
{
DatanodeProtocolClientSideTranslatorPB mock = Org.Mockito.Mockito.Mock <DatanodeProtocolClientSideTranslatorPB
>();
Org.Mockito.Mockito.DoReturn(new NamespaceInfo(1, FakeClusterid, FakeBpid, 0)).When
(mock).VersionRequest();
Org.Mockito.Mockito.DoReturn(DFSTestUtil.GetLocalDatanodeRegistration()).When(mock
).RegisterDatanode(Org.Mockito.Mockito.Any <DatanodeRegistration>());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.HeartbeatAnswer(this, nnIdx))
.When(mock).SendHeartbeat(Org.Mockito.Mockito.Any <DatanodeRegistration>(), Org.Mockito.Mockito
.Any <StorageReport[]>(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.AnyLong
(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito
.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary>());
mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.
Standby, 0);
return(mock);
}
public virtual void TestReportBadBlocksWhenNNThrowsStandbyException()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
// Return nothing when active Active Namenode calls reportBadBlocks
Org.Mockito.Mockito.DoNothing().When(mockNN1).ReportBadBlocks(Org.Mockito.Mockito
.Any <LocatedBlock[]>());
RemoteException re = new RemoteException(typeof(StandbyException).FullName, "Operation category WRITE is not supported in state "
+ "standby", RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto.ErrorApplication
);
// Return StandbyException wrapped in RemoteException when Standby NN
// calls reportBadBlocks
Org.Mockito.Mockito.DoThrow(re).When(mockNN2).ReportBadBlocks(Org.Mockito.Mockito
.Any <LocatedBlock[]>());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
// Send heartbeat so that the BpServiceActor can report bad block to
// namenode
bpos.TriggerHeartbeatForTests();
Org.Mockito.Mockito.Verify(mockNN2, Org.Mockito.Mockito.Times(1)).ReportBadBlocks
(Org.Mockito.Mockito.Any <LocatedBlock[]>());
// Trigger another heartbeat, this will send reportBadBlock again if it
// is present in the queue.
bpos.TriggerHeartbeatForTests();
Org.Mockito.Mockito.Verify(mockNN2, Org.Mockito.Mockito.Times(1)).ReportBadBlocks
(Org.Mockito.Mockito.Any <LocatedBlock[]>());
}
finally
{
bpos.Stop();
}
}
public virtual void TestPickActiveNameNode()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
// NN2 claims active at a higher txid
mockHaStatuses[1] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 2);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN2, bpos.GetActiveNN());
// Even after another heartbeat from the first NN, it should
// think NN2 is active, since it claimed a higher txid
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN2, bpos.GetActiveNN());
// Even if NN2 goes to standby, DN shouldn't reset to talking to NN1,
// because NN1's txid is lower than the last active txid. Instead,
// it should consider neither active.
mockHaStatuses[1] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Standby
, 2);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Now if NN1 goes back to a higher txid, it should be considered active
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 3);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
}
finally
{
bpos.Stop();
}
}
public virtual void TestReportBadBlockWhenStandbyNNTimesOut()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 0)).When(mockNN1).ReportBadBlocks(Org.Mockito.Mockito.Any <LocatedBlock[]>
());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 1)).When(mockNN2).ReportBadBlocks(Org.Mockito.Mockito.Any <LocatedBlock[]>
());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
Sharpen.Thread.Sleep(10000);
long difference = secondCallTime - firstCallTime;
NUnit.Framework.Assert.IsTrue("Active namenode reportBadBlock processing should be "
+ "independent of standby namenode reportBadBlock processing ", difference < 5000
);
}
finally
{
bpos.Stop();
}
}
public virtual void TestTrySendErrorReportWhenStandbyNNTimesOut()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 0)).When(mockNN1).ErrorReport(Org.Mockito.Mockito.Any <DatanodeRegistration
>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyString());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 1)).When(mockNN2).ErrorReport(Org.Mockito.Mockito.Any <DatanodeRegistration
>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyString());
string errorString = "Can't send invalid block " + FakeBlock;
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
Sharpen.Thread.Sleep(10000);
long difference = secondCallTime - firstCallTime;
NUnit.Framework.Assert.IsTrue("Active namenode trySendErrorReport processing " +
"should be independent of standby namenode trySendErrorReport" + " processing ",
difference < 5000);
}
finally
{
bpos.Stop();
}
}
