/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void TriggerHeartbeats(IList <DataNode> datanodes)
{
foreach (DataNode dn in datanodes)
{
DataNodeTestUtils.TriggerHeartbeat(dn);
}
Sharpen.Thread.Sleep(100);
}
public virtual void TestNoSpaceArchive()
{
Log.Info("testNoSpaceArchive");
TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap
(0);
TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme();
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
try
{
test.RunBasicTest(false);
// create 2 hot files with replication 3
short replication = 3;
for (int i = 0; i < 2; i++)
{
Path p = new Path(pathPolicyMap.cold, "file" + i);
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L);
WaitForAllReplicas(replication, p, test.dfs);
}
// set all the ARCHIVE volume to full
foreach (DataNode dn in test.cluster.GetDataNodes())
{
SetVolumeFull(dn, StorageType.Archive);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
{
// test increasing replication but new replicas cannot be created
// since no more ARCHIVE space.
Path file0 = new Path(pathPolicyMap.cold, "file0");
TestStorageMover.Replication r = test.GetReplication(file0);
NUnit.Framework.Assert.AreEqual(0, r.disk);
short newReplication = (short)5;
test.dfs.SetReplication(file0, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(file0, 0, r.archive);
}
{
// test creating a hot file
Path p = new Path(pathPolicyMap.hot, "foo");
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, (short)3, 0L);
}
{
//test move a cold file to warm
Path file1 = new Path(pathPolicyMap.cold, "file1");
test.dfs.Rename(file1, pathPolicyMap.warm);
test.Migrate();
test.Verify(true);
}
}
finally
{
test.ShutdownCluster();
}
}
public virtual void TestNoSpaceDisk()
{
Log.Info("testNoSpaceDisk");
TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap
(0);
TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme();
Configuration conf = new Configuration(DefaultConf);
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(conf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
try
{
test.RunBasicTest(false);
// create 2 hot files with replication 3
short replication = 3;
for (int i = 0; i < 2; i++)
{
Path p = new Path(pathPolicyMap.hot, "file" + i);
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L);
WaitForAllReplicas(replication, p, test.dfs);
}
// set all the DISK volume to full
foreach (DataNode dn in test.cluster.GetDataNodes())
{
SetVolumeFull(dn, StorageType.Disk);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
// test increasing replication. Since DISK is full,
// new replicas should be stored in ARCHIVE as a fallback storage.
Path file0 = new Path(pathPolicyMap.hot, "file0");
TestStorageMover.Replication r = test.GetReplication(file0);
short newReplication = (short)5;
test.dfs.SetReplication(file0, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(file0, r.disk, newReplication - r.disk);
// test creating a cold file and then increase replication
Path p_1 = new Path(pathPolicyMap.cold, "foo");
DFSTestUtil.CreateFile(test.dfs, p_1, BlockSize, replication, 0L);
test.VerifyReplication(p_1, 0, replication);
test.dfs.SetReplication(p_1, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(p_1, 0, newReplication);
//test move a hot file to warm
Path file1 = new Path(pathPolicyMap.hot, "file1");
test.dfs.Rename(file1, pathPolicyMap.warm);
test.Migrate();
test.VerifyFile(new Path(pathPolicyMap.warm, "file1"), Warm.GetId());
}
finally
{
test.ShutdownCluster();
}
}
/// <exception cref="System.Exception"/>
public override void DoAnAction()
{
foreach (DataNode dn in this._enclosing.cluster.GetDataNodes())
{
DataNodeTestUtils.TriggerDeletionReport(dn);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
for (int i = 0; i < 2; i++)
{
NameNode nn = this._enclosing.cluster.GetNameNode(i);
BlockManagerTestUtil.ComputeAllPendingWork(nn.GetNamesystem().GetBlockManager());
}
Sharpen.Thread.Sleep(interval);
}
public virtual void TestBlockReceived()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
string file = "/tmp.txt";
Path filePath = new Path(file);
short replFactor = 1;
DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
// temporarily stop the heartbeat
AList <DataNode> datanodes = cluster.GetDataNodes();
for (int i = 0; i < DatanodeCount; i++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
}
hdfs.SetReplication(filePath, (short)DatanodeCount);
BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
Block[] blocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
DatanodeInfo existingDn = (locatedBlock.GetLocations())[0];
int reportDnNum = 0;
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// let two datanodes (other than the one that already has the data) to
// report to NN
for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
{
if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// let the same datanodes report again
for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
{
if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// re-enable heartbeat for the datanode that has data
for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
}
Sharpen.Thread.Sleep(5000);
NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestXceiverCount()
{
Configuration conf = new HdfsConfiguration();
// retry one time, if close fails
conf.SetInt(DFSConfigKeys.DfsClientBlockWriteLocatefollowingblockRetriesKey, 1);
MiniDFSCluster cluster = null;
int nodes = 8;
int fileCount = 5;
short fileRepl = 3;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nodes).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager();
IList <DataNode> datanodes = cluster.GetDataNodes();
DistributedFileSystem fs = cluster.GetFileSystem();
// trigger heartbeats in case not already sent
TriggerHeartbeats(datanodes);
// check that all nodes are live and in service
int expectedTotalLoad = nodes;
// xceiver server adds 1 to load
int expectedInServiceNodes = nodes;
int expectedInServiceLoad = nodes;
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// shutdown half the nodes and force a heartbeat check to ensure
// counts are accurate
for (int i = 0; i < nodes / 2; i++)
{
DataNode dn = datanodes[i];
DatanodeDescriptor dnd = dnm.GetDatanode(dn.GetDatanodeId());
dn.Shutdown();
DFSTestUtil.SetDatanodeDead(dnd);
BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager());
//Verify decommission of dead node won't impact nodesInService metrics.
dnm.GetDecomManager().StartDecommission(dnd);
expectedInServiceNodes--;
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, namesystem.GetNumLiveDataNodes
());
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
//Verify recommission of dead node won't impact nodesInService metrics.
dnm.GetDecomManager().StopDecommission(dnd);
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
}
// restart the nodes to verify that counts are correct after
// node re-registration
cluster.RestartDataNodes();
cluster.WaitActive();
datanodes = cluster.GetDataNodes();
expectedInServiceNodes = nodes;
NUnit.Framework.Assert.AreEqual(nodes, datanodes.Count);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// create streams and hsync to force datastreamers to start
DFSOutputStream[] streams = new DFSOutputStream[fileCount];
for (int i_1 = 0; i_1 < fileCount; i_1++)
{
streams[i_1] = (DFSOutputStream)fs.Create(new Path("/f" + i_1), fileRepl).GetWrappedStream
();
streams[i_1].Write(Sharpen.Runtime.GetBytesForString("1"));
streams[i_1].Hsync();
// the load for writers is 2 because both the write xceiver & packet
// responder threads are counted in the load
expectedTotalLoad += 2 * fileRepl;
expectedInServiceLoad += 2 * fileRepl;
}
// force nodes to send load update
TriggerHeartbeats(datanodes);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// decomm a few nodes, substract their load from the expected load,
// trigger heartbeat to force load update
for (int i_2 = 0; i_2 < fileRepl; i_2++)
{
expectedInServiceNodes--;
DatanodeDescriptor dnd = dnm.GetDatanode(datanodes[i_2].GetDatanodeId());
expectedInServiceLoad -= dnd.GetXceiverCount();
dnm.GetDecomManager().StartDecommission(dnd);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_2]);
Sharpen.Thread.Sleep(100);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
}
// check expected load while closing each stream. recalc expected
// load based on whether the nodes in the pipeline are decomm
for (int i_3 = 0; i_3 < fileCount; i_3++)
{
int decomm = 0;
foreach (DatanodeInfo dni in streams[i_3].GetPipeline())
{
DatanodeDescriptor dnd = dnm.GetDatanode(dni);
expectedTotalLoad -= 2;
if (dnd.IsDecommissionInProgress() || dnd.IsDecommissioned())
{
decomm++;
}
else
{
expectedInServiceLoad -= 2;
}
}
try
{
streams[i_3].Close();
}
catch (IOException ioe)
{
// nodes will go decommissioned even if there's a UC block whose
// other locations are decommissioned too. we'll ignore that
// bug for now
if (decomm < fileRepl)
{
throw;
}
}
TriggerHeartbeats(datanodes);
// verify node count and loads
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
}
// shutdown each node, verify node counts based on decomm state
for (int i_4 = 0; i_4 < nodes; i_4++)
{
DataNode dn = datanodes[i_4];
dn.Shutdown();
// force it to appear dead so live count decreases
DatanodeDescriptor dnDesc = dnm.GetDatanode(dn.GetDatanodeId());
DFSTestUtil.SetDatanodeDead(dnDesc);
BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager());
NUnit.Framework.Assert.AreEqual(nodes - 1 - i_4, namesystem.GetNumLiveDataNodes()
);
// first few nodes are already out of service
if (i_4 >= fileRepl)
{
expectedInServiceNodes--;
}
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
// live nodes always report load of 1. no nodes is load 0
double expectedXceiverAvg = (i_4 == nodes - 1) ? 0.0 : 1.0;
NUnit.Framework.Assert.AreEqual((double)expectedXceiverAvg, GetInServiceXceiverAverage
(namesystem), Epsilon);
}
// final sanity check
CheckClusterHealth(0, namesystem, 0.0, 0, 0.0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}