public virtual void TestHeartbeat()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg);
string storageID = DatanodeStorage.GenerateUuid();
dd.UpdateStorage(new DatanodeStorage(storageID));
int RemainingBlocks = 1;
int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey
, 2);
int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault;
int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks;
int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks;
DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID
) };
try
{
namesystem.WriteLock();
lock (hm)
{
for (int i = 0; i < MaxReplicateBlocks; i++)
{
dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget
);
}
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks);
for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++)
{
blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp));
}
dd.AddBlocksToBeInvalidated(blockList);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction
());
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(0, cmds.Length);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestHeartbeatBlockRecovery()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg1 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[0], poolId);
DatanodeDescriptor dd1 = NameNodeAdapter.GetDatanode(namesystem, nodeReg1);
dd1.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg2 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[1], poolId);
DatanodeDescriptor dd2 = NameNodeAdapter.GetDatanode(namesystem, nodeReg2);
dd2.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg3 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[2], poolId);
DatanodeDescriptor dd3 = NameNodeAdapter.GetDatanode(namesystem, nodeReg3);
dd3.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
try
{
namesystem.WriteLock();
lock (hm)
{
NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg2, dd2, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg3, dd3, namesystem);
// Test with all alive nodes.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
DatanodeStorageInfo[] storages = new DatanodeStorageInfo[] { dd1.GetStorageInfos(
)[0], dd2.GetStorageInfos()[0], dd3.GetStorageInfos()[0] };
BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction
(new Block(0, 0, GenerationStamp.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState
.UnderRecovery, storages);
dd1.AddBlockToBeRecovered(blockInfo);
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem)
.GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
BlockRecoveryCommand recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
DatanodeInfo[] recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
// Test with one stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(2, recoveringNodes.Length);
// dd2 is skipped.
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd3);
// Test with all stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, -60 * 1000);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, -80 * 1000);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
// Only dd1 is included since it heart beated and hence its not stale
// when the list of recovery blocks is constructed.
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}