public virtual void TestStorageWithRemainingCapacity()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = FileSystem.Get(conf);
Path file1 = null;
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg);
// By default, MiniDFSCluster will create 1 datanode with 2 storages.
// Assigning 64k for remaining storage capacity and will
//create a file with 100k.
foreach (DatanodeStorageInfo storage in dd.GetStorageInfos())
{
storage.SetUtilizationForTesting(65536, 0, 65536, 0);
}
//sum of the remaining capacity of both the storages
dd.SetRemaining(131072);
file1 = new Path("testRemainingStorage.dat");
try
{
DFSTestUtil.CreateFile(fs, file1, 102400, 102400, 102400, (short)1, unchecked ((int
)(0x1BAD5EED)));
}
catch (RemoteException re)
{
GenericTestUtils.AssertExceptionContains("nodes instead of " + "minReplication",
re);
}
}
finally
{
// Clean up
NUnit.Framework.Assert.IsTrue(fs.Exists(file1));
fs.Delete(file1, true);
NUnit.Framework.Assert.IsTrue(!fs.Exists(file1));
cluster.Shutdown();
}
}
public virtual void TestHeartbeatBlockRecovery()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg1 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[0], poolId);
DatanodeDescriptor dd1 = NameNodeAdapter.GetDatanode(namesystem, nodeReg1);
dd1.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg2 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[1], poolId);
DatanodeDescriptor dd2 = NameNodeAdapter.GetDatanode(namesystem, nodeReg2);
dd2.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
DatanodeRegistration nodeReg3 = DataNodeTestUtils.GetDNRegistrationForBP(cluster.
GetDataNodes()[2], poolId);
DatanodeDescriptor dd3 = NameNodeAdapter.GetDatanode(namesystem, nodeReg3);
dd3.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid()));
try
{
namesystem.WriteLock();
lock (hm)
{
NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg2, dd2, namesystem);
NameNodeAdapter.SendHeartBeat(nodeReg3, dd3, namesystem);
// Test with all alive nodes.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, 0);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
DatanodeStorageInfo[] storages = new DatanodeStorageInfo[] { dd1.GetStorageInfos(
)[0], dd2.GetStorageInfos()[0], dd3.GetStorageInfos()[0] };
BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction
(new Block(0, 0, GenerationStamp.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState
.UnderRecovery, storages);
dd1.AddBlockToBeRecovered(blockInfo);
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem)
.GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
BlockRecoveryCommand recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
DatanodeInfo[] recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
// Test with one stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
NUnit.Framework.Assert.AreEqual(2, recoveringNodes.Length);
// dd2 is skipped.
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd3);
// Test with all stale node.
DFSTestUtil.ResetLastUpdatesWithOffset(dd1, -60 * 1000);
// More than the default stale interval of 30 seconds.
DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000);
DFSTestUtil.ResetLastUpdatesWithOffset(dd3, -80 * 1000);
blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp
.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages
);
dd1.AddBlockToBeRecovered(blockInfo);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction
());
recoveryCommand = (BlockRecoveryCommand)cmds[0];
NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count);
recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks
(), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations();
// Only dd1 is included since it heart beated and hence its not stale
// when the list of recovery blocks is constructed.
NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length);
NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1);
NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2);
NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestHeartbeat()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager
();
string poolId = namesystem.GetBlockPoolId();
DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg);
string storageID = DatanodeStorage.GenerateUuid();
dd.UpdateStorage(new DatanodeStorage(storageID));
int RemainingBlocks = 1;
int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey
, 2);
int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault;
int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks;
int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks;
DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID
) };
try
{
namesystem.WriteLock();
lock (hm)
{
for (int i = 0; i < MaxReplicateBlocks; i++)
{
dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget
);
}
DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks);
for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++)
{
blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp));
}
dd.AddBlocksToBeInvalidated(blockList);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(2, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction()
);
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction
());
NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(1, cmds.Length);
NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction
());
NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks
().Length);
cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands();
NUnit.Framework.Assert.AreEqual(0, cmds.Length);
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDeletedBlockWhenAddBlockIsInEdit()
{
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
DFSClient client = null;
try
{
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual("Number of namenodes is not 2", 2, cluster.GetNumNameNodes
());
// Transitioning the namenode 0 to active.
cluster.TransitionToActive(0);
NUnit.Framework.Assert.IsTrue("Namenode 0 should be in active state", cluster.GetNameNode
(0).IsActiveState());
NUnit.Framework.Assert.IsTrue("Namenode 1 should be in standby state", cluster.GetNameNode
(1).IsStandbyState());
// Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
// to true.
DataNodeTestUtils.TriggerHeartbeat(cluster.GetDataNodes()[0]);
FileSystem fs = cluster.GetFileSystem(0);
// Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
// to false.
cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
(false).Build());
Path fileName = new Path("/tmp.txt");
// create a file with one block
DFSTestUtil.CreateFile(fs, fileName, 10L, (short)1, 1234L);
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
client = new DFSClient(cluster.GetFileSystem(0).GetUri(), conf);
IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, 10L).GetLocatedBlocks();
NUnit.Framework.Assert.IsTrue(locatedBlocks.Count == 1);
NUnit.Framework.Assert.IsTrue(locatedBlocks[0].GetLocations().Length == 1);
// add a second datanode to the cluster
cluster.StartDataNodes(conf, 1, true, null, null, null, null);
NUnit.Framework.Assert.AreEqual("Number of datanodes should be 2", 2, cluster.GetDataNodes
().Count);
DataNode dn0 = cluster.GetDataNodes()[0];
DataNode dn1 = cluster.GetDataNodes()[1];
string activeNNBPId = cluster.GetNamesystem(0).GetBlockPoolId();
DatanodeDescriptor sourceDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
(0), dn0.GetDNRegistrationForBP(activeNNBPId));
DatanodeDescriptor destDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
(0), dn1.GetDNRegistrationForBP(activeNNBPId));
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
Log.Info("replaceBlock: " + ReplaceBlock(block, (DatanodeInfo)sourceDnDesc, (DatanodeInfo
)sourceDnDesc, (DatanodeInfo)destDnDesc));
// Waiting for the FsDatasetAsyncDsikService to delete the block
Sharpen.Thread.Sleep(3000);
// Triggering the incremental block report to report the deleted block to
// namnemode
cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
(true).Build());
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.IsTrue("Namenode 1 should be in active state", cluster.GetNameNode
(1).IsActiveState());
NUnit.Framework.Assert.IsTrue("Namenode 0 should be in standby state", cluster.GetNameNode
(0).IsStandbyState());
client.Close();
// Opening a new client for new active namenode
client = new DFSClient(cluster.GetFileSystem(1).GetUri(), conf);
IList <LocatedBlock> locatedBlocks1 = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, 10L).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, locatedBlocks1.Count);
NUnit.Framework.Assert.AreEqual("The block should be only on 1 datanode ", 1, locatedBlocks1
[0].GetLocations().Length);
}
finally
{
IOUtils.Cleanup(null, client);
cluster.Shutdown();
}
}
