/// <summary>Inject a fake 'deleted' block into the BPServiceActor state.</summary>
private void InjectBlockDeleted()
{
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(GetDummyBlock(), ReceivedDeletedBlockInfo.BlockStatus
.DeletedBlock, null);
actor.NotifyNamenodeDeletedBlock(rdbi, storageUuid);
}
public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
private static StorageReceivedDeletedBlocks[] MakeReportForReceivedBlock(Block block
, DatanodeStorage storage)
{
ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1];
receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, null);
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1];
reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks);
return(reports);
}
internal virtual void NotifyNamenodeReceivingBlock(ExtendedBlock block, string storageUuid
)
{
CheckBlock(block);
ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(block.GetLocalBlock
(), ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null);
foreach (BPServiceActor actor in bpServices)
{
actor.NotifyNamenodeBlock(bInfo, storageUuid, false);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void VerifyIncrementalBlockReports(bool splitReports)
{
// Get the block list for the file with the block locations.
LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName());
// We will send 'fake' incremental block reports to the NN that look
// like they originated from DN 0.
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[dn0.GetFSDataset
().GetVolumes().Count];
// Lie to the NN that one block on each storage has been deleted.
for (int i = 0; i < reports.Length; ++i)
{
FsVolumeSpi volume = dn0.GetFSDataset().GetVolumes()[i];
bool foundBlockOnStorage = false;
ReceivedDeletedBlockInfo[] rdbi = new ReceivedDeletedBlockInfo[1];
// Find the first block on this storage and mark it as deleted for the
// report.
foreach (LocatedBlock block in blocks.GetLocatedBlocks())
{
if (block.GetStorageIDs()[0].Equals(volume.GetStorageID()))
{
rdbi[0] = new ReceivedDeletedBlockInfo(block.GetBlock().GetLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus
.DeletedBlock, null);
foundBlockOnStorage = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(foundBlockOnStorage);
reports[i] = new StorageReceivedDeletedBlocks(volume.GetStorageID(), rdbi);
if (splitReports)
{
// If we are splitting reports then send the report for this storage now.
StorageReceivedDeletedBlocks[] singletonReport = new StorageReceivedDeletedBlocks
[] { reports[i] };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, singletonReport);
}
}
if (!splitReports)
{
// Send a combined report.
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports);
}
// Make sure that the deleted block from each storage was picked up
// by the NameNode.
Assert.AssertThat(cluster.GetNamesystem().GetMissingBlocksCount(), IS.Is((long)reports
.Length));
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.BlockReceivedAndDeletedResponseProto BlockReceivedAndDeleted
(RpcController controller, DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto
request)
{
IList <DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto> sBlocks = request
.GetBlocksList();
StorageReceivedDeletedBlocks[] info = new StorageReceivedDeletedBlocks[sBlocks.Count
];
for (int i = 0; i < sBlocks.Count; i++)
{
DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto sBlock = sBlocks[i];
IList <DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto> list = sBlock.GetBlocksList
();
ReceivedDeletedBlockInfo[] rdBlocks = new ReceivedDeletedBlockInfo[list.Count];
for (int j = 0; j < list.Count; j++)
{
rdBlocks[j] = PBHelper.Convert(list[j]);
}
if (sBlock.HasStorage())
{
info[i] = new StorageReceivedDeletedBlocks(PBHelper.Convert(sBlock.GetStorage()),
rdBlocks);
}
else
{
info[i] = new StorageReceivedDeletedBlocks(sBlock.GetStorageUuid(), rdBlocks);
}
}
try
{
impl.BlockReceivedAndDeleted(PBHelper.Convert(request.GetRegistration()), request
.GetBlockPoolId(), info);
}
catch (IOException e)
{
throw new ServiceException(e);
}
return(VoidBlockReceivedAndDeleteResponse);
}
/// <exception cref="System.Exception"/>
private void TestTriggerBlockReport(bool incremental)
{
Configuration conf = new HdfsConfiguration();
// Set a really long value for dfs.blockreport.intervalMsec and
// dfs.heartbeat.interval, so that incremental block reports and heartbeats
// won't be sent during this test unless they're triggered
// manually.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10800000L);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1080L);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(cluster
.GetDataNodes()[0], cluster.GetNameNode());
DFSTestUtil.CreateFile(fs, new Path("/abc"), 16, (short)1, 1L);
// We should get 1 incremental block report.
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// We should not receive any more incremental or incremental block reports,
// since the interval we configured is so long.
for (int i = 0; i < 3; i++)
{
Sharpen.Thread.Sleep(10);
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(0)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
// Create a fake block deletion notification on the DataNode.
// This will be sent with the next incremental block report.
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(new Block(5678, 512,
1000), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null);
DataNode datanode = cluster.GetDataNodes()[0];
BPServiceActor actor = datanode.GetAllBpOs()[0].GetBPServiceActors()[0];
string storageUuid = datanode.GetFSDataset().GetVolumes()[0].GetStorageID();
actor.NotifyNamenodeDeletedBlock(rdbi, storageUuid);
// Manually trigger a block report.
datanode.TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental(incremental
).Build());
// triggerBlockReport returns before the block report is
// actually sent. Wait for it to be sent here.
if (incremental)
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(2)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
else
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
}
cluster.Shutdown();
}
