public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
private static StorageReceivedDeletedBlocks[] MakeReportForReceivedBlock(Block block
, DatanodeStorage storage)
{
ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1];
receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, null);
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1];
reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks);
return(reports);
}
/// <exception cref="System.IO.IOException"/>
public virtual void VerifyIncrementalBlockReports(bool splitReports)
{
// Get the block list for the file with the block locations.
LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName());
// We will send 'fake' incremental block reports to the NN that look
// like they originated from DN 0.
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[dn0.GetFSDataset
().GetVolumes().Count];
// Lie to the NN that one block on each storage has been deleted.
for (int i = 0; i < reports.Length; ++i)
{
FsVolumeSpi volume = dn0.GetFSDataset().GetVolumes()[i];
bool foundBlockOnStorage = false;
ReceivedDeletedBlockInfo[] rdbi = new ReceivedDeletedBlockInfo[1];
// Find the first block on this storage and mark it as deleted for the
// report.
foreach (LocatedBlock block in blocks.GetLocatedBlocks())
{
if (block.GetStorageIDs()[0].Equals(volume.GetStorageID()))
{
rdbi[0] = new ReceivedDeletedBlockInfo(block.GetBlock().GetLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus
.DeletedBlock, null);
foundBlockOnStorage = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(foundBlockOnStorage);
reports[i] = new StorageReceivedDeletedBlocks(volume.GetStorageID(), rdbi);
if (splitReports)
{
// If we are splitting reports then send the report for this storage now.
StorageReceivedDeletedBlocks[] singletonReport = new StorageReceivedDeletedBlocks
[] { reports[i] };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, singletonReport);
}
}
if (!splitReports)
{
// Send a combined report.
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports);
}
// Make sure that the deleted block from each storage was picked up
// by the NameNode.
Assert.AssertThat(cluster.GetNamesystem().GetMissingBlocksCount(), IS.Is((long)reports
.Length));
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.BlockReceivedAndDeletedResponseProto BlockReceivedAndDeleted
(RpcController controller, DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto
request)
{
IList <DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto> sBlocks = request
.GetBlocksList();
StorageReceivedDeletedBlocks[] info = new StorageReceivedDeletedBlocks[sBlocks.Count
];
for (int i = 0; i < sBlocks.Count; i++)
{
DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto sBlock = sBlocks[i];
IList <DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto> list = sBlock.GetBlocksList
();
ReceivedDeletedBlockInfo[] rdBlocks = new ReceivedDeletedBlockInfo[list.Count];
for (int j = 0; j < list.Count; j++)
{
rdBlocks[j] = PBHelper.Convert(list[j]);
}
if (sBlock.HasStorage())
{
info[i] = new StorageReceivedDeletedBlocks(PBHelper.Convert(sBlock.GetStorage()),
rdBlocks);
}
else
{
info[i] = new StorageReceivedDeletedBlocks(sBlock.GetStorageUuid(), rdBlocks);
}
}
try
{
impl.BlockReceivedAndDeleted(PBHelper.Convert(request.GetRegistration()), request
.GetBlockPoolId(), info);
}
catch (IOException e)
{
throw new ServiceException(e);
}
return(VoidBlockReceivedAndDeleteResponse);
}
public virtual void TestBlockReceived()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
string file = "/tmp.txt";
Path filePath = new Path(file);
short replFactor = 1;
DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
// temporarily stop the heartbeat
AList <DataNode> datanodes = cluster.GetDataNodes();
for (int i = 0; i < DatanodeCount; i++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
}
hdfs.SetReplication(filePath, (short)DatanodeCount);
BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
Block[] blocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
DatanodeInfo existingDn = (locatedBlock.GetLocations())[0];
int reportDnNum = 0;
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// let two datanodes (other than the one that already has the data) to
// report to NN
for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
{
if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// let the same datanodes report again
for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
{
if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// re-enable heartbeat for the datanode that has data
for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
}
Sharpen.Thread.Sleep(5000);
NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSafeModeIBRBeforeFirstFullBR()
{
// pretend to be in safemode
Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode();
DatanodeDescriptor node = nodes[0];
DatanodeStorageInfo ds = node.GetStorageInfos()[0];
node.isAlive = true;
DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty
);
// register new node
bm.GetDatanodeManager().RegisterDatanode(nodeReg);
bm.GetDatanodeManager().AddDatanode(node);
NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node));
NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount());
// Build a incremental report
IList <ReceivedDeletedBlockInfo> rdbiList = new AList <ReceivedDeletedBlockInfo>();
// Build a full report
BlockListAsLongs.Builder builder = BlockListAsLongs.Builder();
// blk_42 is finalized.
long receivedBlockId = 42;
// arbitrary
BlockInfoContiguous receivedBlock = AddBlockToBM(receivedBlockId);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivedBlock), ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, null));
builder.Add(new FinalizedReplica(receivedBlock, null, null));
// blk_43 is under construction.
long receivingBlockId = 43;
BlockInfoContiguous receivingBlock = AddUcBlockToBM(receivingBlockId);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingBlock), ReceivedDeletedBlockInfo.BlockStatus
.ReceivingBlock, null));
builder.Add(new ReplicaBeingWritten(receivingBlock, null, null, null));
// blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
long receivingReceivedBlockId = 44;
BlockInfoContiguous receivingReceivedBlock = AddBlockToBM(receivingReceivedBlockId
);
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null));
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null));
builder.Add(new FinalizedReplica(receivingReceivedBlock, null, null));
// blk_45 is not in full BR, because it's deleted.
long ReceivedDeletedBlockId = 45;
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId),
ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null));
rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId),
ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null));
// blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
long existedBlockId = 46;
BlockInfoContiguous existedBlock = AddBlockToBM(existedBlockId);
builder.Add(new FinalizedReplica(existedBlock, null, null));
// process IBR and full BR
StorageReceivedDeletedBlocks srdb = new StorageReceivedDeletedBlocks(new DatanodeStorage
(ds.GetStorageID()), Sharpen.Collections.ToArray(rdbiList, new ReceivedDeletedBlockInfo
[rdbiList.Count]));
bm.ProcessIncrementalBlockReport(node, srdb);
// Make sure it's the first full report
NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount());
bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), builder.Build(), null
, false);
NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount());
// verify the storage info is correct
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivedBlockId)).FindStorageInfo
(ds) >= 0);
NUnit.Framework.Assert.IsTrue(((BlockInfoContiguousUnderConstruction)bm.GetStoredBlock
(new Block(receivingBlockId))).GetNumExpectedLocations() > 0);
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivingReceivedBlockId
)).FindStorageInfo(ds) >= 0);
NUnit.Framework.Assert.IsNull(bm.GetStoredBlock(new Block(ReceivedDeletedBlockId)
));
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(existedBlock)).FindStorageInfo
(ds) >= 0);
}
