public virtual void TestTrySendErrorReportWhenNNThrowsIOException()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new _Answer_602(this)).When(mockNN1).ErrorReport(Org.Mockito.Mockito
.Any <DatanodeRegistration>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.
AnyString());
// Throw an IOException when this function is first called which will
// in turn add that errorReport back to the bpThreadQueue and let it
// process the next time.
string errorString = "Can't send invalid block " + FakeBlock;
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
Sharpen.Thread.Sleep(10000);
NUnit.Framework.Assert.IsTrue("Active namenode didn't add the report back to the queue "
+ "when errorReport threw IOException", secondCallTime != 0);
}
finally
{
bpos.Stop();
}
}
public virtual void TestReportBadBlocksWhenNNThrowsStandbyException()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
// Return nothing when active Active Namenode calls reportBadBlocks
Org.Mockito.Mockito.DoNothing().When(mockNN1).ReportBadBlocks(Org.Mockito.Mockito
.Any <LocatedBlock[]>());
RemoteException re = new RemoteException(typeof(StandbyException).FullName, "Operation category WRITE is not supported in state "
+ "standby", RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto.ErrorApplication
);
// Return StandbyException wrapped in RemoteException when Standby NN
// calls reportBadBlocks
Org.Mockito.Mockito.DoThrow(re).When(mockNN2).ReportBadBlocks(Org.Mockito.Mockito
.Any <LocatedBlock[]>());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
// Send heartbeat so that the BpServiceActor can report bad block to
// namenode
bpos.TriggerHeartbeatForTests();
Org.Mockito.Mockito.Verify(mockNN2, Org.Mockito.Mockito.Times(1)).ReportBadBlocks
(Org.Mockito.Mockito.Any <LocatedBlock[]>());
// Trigger another heartbeat, this will send reportBadBlock again if it
// is present in the queue.
bpos.TriggerHeartbeatForTests();
Org.Mockito.Mockito.Verify(mockNN2, Org.Mockito.Mockito.Times(1)).ReportBadBlocks
(Org.Mockito.Mockito.Any <LocatedBlock[]>());
}
finally
{
bpos.Stop();
}
}
public virtual void TestReportBadBlockWhenStandbyNNTimesOut()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 0)).When(mockNN1).ReportBadBlocks(Org.Mockito.Mockito.Any <LocatedBlock[]>
());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 1)).When(mockNN2).ReportBadBlocks(Org.Mockito.Mockito.Any <LocatedBlock[]>
());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
bpos.ReportBadBlocks(FakeBlock, mockFSDataset.GetVolume(FakeBlock).GetStorageID()
, mockFSDataset.GetVolume(FakeBlock).GetStorageType());
Sharpen.Thread.Sleep(10000);
long difference = secondCallTime - firstCallTime;
NUnit.Framework.Assert.IsTrue("Active namenode reportBadBlock processing should be "
+ "independent of standby namenode reportBadBlock processing ", difference < 5000
);
}
finally
{
bpos.Stop();
}
}
public virtual void TestTrySendErrorReportWhenStandbyNNTimesOut()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
// Now mockNN1 is acting like active namenode and mockNN2 as Standby
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 0)).When(mockNN1).ErrorReport(Org.Mockito.Mockito.Any <DatanodeRegistration
>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyString());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.BPOfferServiceSynchronousCallAnswer
(this, 1)).When(mockNN2).ErrorReport(Org.Mockito.Mockito.Any <DatanodeRegistration
>(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyString());
string errorString = "Can't send invalid block " + FakeBlock;
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
bpos.TrySendErrorReport(DatanodeProtocol.InvalidBlock, errorString);
Sharpen.Thread.Sleep(10000);
long difference = secondCallTime - firstCallTime;
NUnit.Framework.Assert.IsTrue("Active namenode trySendErrorReport processing " +
"should be independent of standby namenode trySendErrorReport" + " processing ",
difference < 5000);
}
finally
{
bpos.Stop();
}
}
public virtual void TestPickActiveNameNode()
{
BPOfferService bpos = SetupBPOSForNNs(mockNN1, mockNN2);
bpos.Start();
try
{
WaitForInitialization(bpos);
// Should start with neither NN as active.
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Have NN1 claim active at txid 1
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 1);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
// NN2 claims active at a higher txid
mockHaStatuses[1] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 2);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN2, bpos.GetActiveNN());
// Even after another heartbeat from the first NN, it should
// think NN2 is active, since it claimed a higher txid
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN2, bpos.GetActiveNN());
// Even if NN2 goes to standby, DN shouldn't reset to talking to NN1,
// because NN1's txid is lower than the last active txid. Instead,
// it should consider neither active.
mockHaStatuses[1] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Standby
, 2);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.IsNull(bpos.GetActiveNN());
// Now if NN1 goes back to a higher txid, it should be considered active
mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.Active
, 3);
bpos.TriggerHeartbeatForTests();
NUnit.Framework.Assert.AreSame(mockNN1, bpos.GetActiveNN());
}
finally
{
bpos.Stop();
}
}