/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void TriggerHeartbeats(IList <DataNode> datanodes) { foreach (DataNode dn in datanodes) { DataNodeTestUtils.TriggerHeartbeat(dn); } Sharpen.Thread.Sleep(100); }
public virtual void TestNoSpaceArchive() { Log.Info("testNoSpaceArchive"); TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap (0); TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme(); TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme (DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null); TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme , nsScheme); try { test.RunBasicTest(false); // create 2 hot files with replication 3 short replication = 3; for (int i = 0; i < 2; i++) { Path p = new Path(pathPolicyMap.cold, "file" + i); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L); WaitForAllReplicas(replication, p, test.dfs); } // set all the ARCHIVE volume to full foreach (DataNode dn in test.cluster.GetDataNodes()) { SetVolumeFull(dn, StorageType.Archive); DataNodeTestUtils.TriggerHeartbeat(dn); } { // test increasing replication but new replicas cannot be created // since no more ARCHIVE space. Path file0 = new Path(pathPolicyMap.cold, "file0"); TestStorageMover.Replication r = test.GetReplication(file0); NUnit.Framework.Assert.AreEqual(0, r.disk); short newReplication = (short)5; test.dfs.SetReplication(file0, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(file0, 0, r.archive); } { // test creating a hot file Path p = new Path(pathPolicyMap.hot, "foo"); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, (short)3, 0L); } { //test move a cold file to warm Path file1 = new Path(pathPolicyMap.cold, "file1"); test.dfs.Rename(file1, pathPolicyMap.warm); test.Migrate(); test.Verify(true); } } finally { test.ShutdownCluster(); } }
public virtual void TestNoSpaceDisk() { Log.Info("testNoSpaceDisk"); TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap (0); TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme(); Configuration conf = new Configuration(DefaultConf); TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme (conf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null); TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme , nsScheme); try { test.RunBasicTest(false); // create 2 hot files with replication 3 short replication = 3; for (int i = 0; i < 2; i++) { Path p = new Path(pathPolicyMap.hot, "file" + i); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L); WaitForAllReplicas(replication, p, test.dfs); } // set all the DISK volume to full foreach (DataNode dn in test.cluster.GetDataNodes()) { SetVolumeFull(dn, StorageType.Disk); DataNodeTestUtils.TriggerHeartbeat(dn); } // test increasing replication. Since DISK is full, // new replicas should be stored in ARCHIVE as a fallback storage. Path file0 = new Path(pathPolicyMap.hot, "file0"); TestStorageMover.Replication r = test.GetReplication(file0); short newReplication = (short)5; test.dfs.SetReplication(file0, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(file0, r.disk, newReplication - r.disk); // test creating a cold file and then increase replication Path p_1 = new Path(pathPolicyMap.cold, "foo"); DFSTestUtil.CreateFile(test.dfs, p_1, BlockSize, replication, 0L); test.VerifyReplication(p_1, 0, replication); test.dfs.SetReplication(p_1, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(p_1, 0, newReplication); //test move a hot file to warm Path file1 = new Path(pathPolicyMap.hot, "file1"); test.dfs.Rename(file1, pathPolicyMap.warm); test.Migrate(); test.VerifyFile(new Path(pathPolicyMap.warm, "file1"), Warm.GetId()); } finally { test.ShutdownCluster(); } }
/// <exception cref="System.Exception"/> public override void DoAnAction() { foreach (DataNode dn in this._enclosing.cluster.GetDataNodes()) { DataNodeTestUtils.TriggerDeletionReport(dn); DataNodeTestUtils.TriggerHeartbeat(dn); } for (int i = 0; i < 2; i++) { NameNode nn = this._enclosing.cluster.GetNameNode(i); BlockManagerTestUtil.ComputeAllPendingWork(nn.GetNamesystem().GetBlockManager()); } Sharpen.Thread.Sleep(interval); }
public virtual void TestBlockReceived() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build(); cluster.WaitActive(); DistributedFileSystem hdfs = cluster.GetFileSystem(); FSNamesystem fsn = cluster.GetNamesystem(); BlockManager blkManager = fsn.GetBlockManager(); string file = "/tmp.txt"; Path filePath = new Path(file); short replFactor = 1; DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0); // temporarily stop the heartbeat AList <DataNode> datanodes = cluster.GetDataNodes(); for (int i = 0; i < DatanodeCount; i++) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true); } hdfs.SetReplication(filePath, (short)DatanodeCount); BlockManagerTestUtil.ComputeAllPendingWork(blkManager); NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size()); INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile(); Block[] blocks = fileNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications .GetNumReplicas(blocks[0])); LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0); DatanodeInfo existingDn = (locatedBlock.GetLocations())[0]; int reportDnNum = 0; string poolId = cluster.GetNamesystem().GetBlockPoolId(); // let two datanodes (other than the one that already has the data) to // report to NN for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++) { if (!datanodes[i_1].GetDatanodeId().Equals(existingDn)) { DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId); StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, string.Empty) }) }; cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report); reportDnNum++; } } NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications .GetNumReplicas(blocks[0])); // let the same datanodes report again for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++) { if (!datanodes[i_2].GetDatanodeId().Equals(existingDn)) { DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId); StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, string.Empty) }) }; cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report); reportDnNum++; } } NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications .GetNumReplicas(blocks[0])); // re-enable heartbeat for the datanode that has data for (int i_3 = 0; i_3 < DatanodeCount; i_3++) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false); DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]); } Sharpen.Thread.Sleep(5000); NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestXceiverCount() { Configuration conf = new HdfsConfiguration(); // retry one time, if close fails conf.SetInt(DFSConfigKeys.DfsClientBlockWriteLocatefollowingblockRetriesKey, 1); MiniDFSCluster cluster = null; int nodes = 8; int fileCount = 5; short fileRepl = 3; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nodes).Build(); cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager(); IList <DataNode> datanodes = cluster.GetDataNodes(); DistributedFileSystem fs = cluster.GetFileSystem(); // trigger heartbeats in case not already sent TriggerHeartbeats(datanodes); // check that all nodes are live and in service int expectedTotalLoad = nodes; // xceiver server adds 1 to load int expectedInServiceNodes = nodes; int expectedInServiceLoad = nodes; CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad); // shutdown half the nodes and force a heartbeat check to ensure // counts are accurate for (int i = 0; i < nodes / 2; i++) { DataNode dn = datanodes[i]; DatanodeDescriptor dnd = dnm.GetDatanode(dn.GetDatanodeId()); dn.Shutdown(); DFSTestUtil.SetDatanodeDead(dnd); BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager()); //Verify decommission of dead node won't impact nodesInService metrics. dnm.GetDecomManager().StartDecommission(dnd); expectedInServiceNodes--; NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, namesystem.GetNumLiveDataNodes ()); NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem )); //Verify recommission of dead node won't impact nodesInService metrics. dnm.GetDecomManager().StopDecommission(dnd); NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem )); } // restart the nodes to verify that counts are correct after // node re-registration cluster.RestartDataNodes(); cluster.WaitActive(); datanodes = cluster.GetDataNodes(); expectedInServiceNodes = nodes; NUnit.Framework.Assert.AreEqual(nodes, datanodes.Count); CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad); // create streams and hsync to force datastreamers to start DFSOutputStream[] streams = new DFSOutputStream[fileCount]; for (int i_1 = 0; i_1 < fileCount; i_1++) { streams[i_1] = (DFSOutputStream)fs.Create(new Path("/f" + i_1), fileRepl).GetWrappedStream (); streams[i_1].Write(Sharpen.Runtime.GetBytesForString("1")); streams[i_1].Hsync(); // the load for writers is 2 because both the write xceiver & packet // responder threads are counted in the load expectedTotalLoad += 2 * fileRepl; expectedInServiceLoad += 2 * fileRepl; } // force nodes to send load update TriggerHeartbeats(datanodes); CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad); // decomm a few nodes, substract their load from the expected load, // trigger heartbeat to force load update for (int i_2 = 0; i_2 < fileRepl; i_2++) { expectedInServiceNodes--; DatanodeDescriptor dnd = dnm.GetDatanode(datanodes[i_2].GetDatanodeId()); expectedInServiceLoad -= dnd.GetXceiverCount(); dnm.GetDecomManager().StartDecommission(dnd); DataNodeTestUtils.TriggerHeartbeat(datanodes[i_2]); Sharpen.Thread.Sleep(100); CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad); } // check expected load while closing each stream. recalc expected // load based on whether the nodes in the pipeline are decomm for (int i_3 = 0; i_3 < fileCount; i_3++) { int decomm = 0; foreach (DatanodeInfo dni in streams[i_3].GetPipeline()) { DatanodeDescriptor dnd = dnm.GetDatanode(dni); expectedTotalLoad -= 2; if (dnd.IsDecommissionInProgress() || dnd.IsDecommissioned()) { decomm++; } else { expectedInServiceLoad -= 2; } } try { streams[i_3].Close(); } catch (IOException ioe) { // nodes will go decommissioned even if there's a UC block whose // other locations are decommissioned too. we'll ignore that // bug for now if (decomm < fileRepl) { throw; } } TriggerHeartbeats(datanodes); // verify node count and loads CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad); } // shutdown each node, verify node counts based on decomm state for (int i_4 = 0; i_4 < nodes; i_4++) { DataNode dn = datanodes[i_4]; dn.Shutdown(); // force it to appear dead so live count decreases DatanodeDescriptor dnDesc = dnm.GetDatanode(dn.GetDatanodeId()); DFSTestUtil.SetDatanodeDead(dnDesc); BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager()); NUnit.Framework.Assert.AreEqual(nodes - 1 - i_4, namesystem.GetNumLiveDataNodes() ); // first few nodes are already out of service if (i_4 >= fileRepl) { expectedInServiceNodes--; } NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem )); // live nodes always report load of 1. no nodes is load 0 double expectedXceiverAvg = (i_4 == nodes - 1) ? 0.0 : 1.0; NUnit.Framework.Assert.AreEqual((double)expectedXceiverAvg, GetInServiceXceiverAverage (namesystem), Epsilon); } // final sanity check CheckClusterHealth(0, namesystem, 0.0, 0, 0.0); } finally { if (cluster != null) { cluster.Shutdown(); } } }