public virtual void TestBlocksCounter() { DatanodeDescriptor dd = BlockManagerTestUtil.GetLocalDatanodeDescriptor(true); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short)1); BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short)2); DatanodeStorageInfo[] storages = dd.GetStorageInfos(); NUnit.Framework.Assert.IsTrue(storages.Length > 0); // add first block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove a non-existent block NUnit.Framework.Assert.IsFalse(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add an existent block NUnit.Framework.Assert.IsFalse(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add second block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk1) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(2, dd.NumBlocks()); // remove first block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove second block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); }
public virtual void TestChooseTargetWithDecomNodes() { namenode.GetNamesystem().WriteLock(); try { string blockPoolId = namenode.GetNamesystem().GetBlockPoolId(); dnManager.HandleHeartbeat(dnrList[3], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[3]), blockPoolId, dataNodes[3].GetCacheCapacity(), dataNodes[3].GetCacheRemaining (), 2, 0, 0, null); dnManager.HandleHeartbeat(dnrList[4], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[4]), blockPoolId, dataNodes[4].GetCacheCapacity(), dataNodes[4].GetCacheRemaining (), 4, 0, 0, null); dnManager.HandleHeartbeat(dnrList[5], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[5]), blockPoolId, dataNodes[5].GetCacheCapacity(), dataNodes[5].GetCacheRemaining (), 4, 0, 0, null); // value in the above heartbeats int load = 2 + 4 + 4; FSNamesystem fsn = namenode.GetNamesystem(); NUnit.Framework.Assert.AreEqual((double)load / 6, dnManager.GetFSClusterStats().GetInServiceXceiverAverage (), Epsilon); // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget() // returns false for (int i = 0; i < 3; i++) { DatanodeDescriptor d = dnManager.GetDatanode(dnrList[i]); dnManager.GetDecomManager().StartDecommission(d); d.SetDecommissioned(); } NUnit.Framework.Assert.AreEqual((double)load / 3, dnManager.GetFSClusterStats().GetInServiceXceiverAverage (), Epsilon); // update references of writer DN to update the de-commissioned state IList <DatanodeDescriptor> liveNodes = new AList <DatanodeDescriptor>(); dnManager.FetchDatanodes(liveNodes, null, false); DatanodeDescriptor writerDn = null; if (liveNodes.Contains(dataNodes[0])) { writerDn = liveNodes[liveNodes.IndexOf(dataNodes[0])]; } // Call chooseTarget() DatanodeStorageInfo[] targets = namenode.GetNamesystem().GetBlockManager().GetBlockPlacementPolicy ().ChooseTarget("testFile.txt", 3, writerDn, new AList <DatanodeStorageInfo>(), false , null, 1024, TestBlockStoragePolicy.DefaultStoragePolicy); NUnit.Framework.Assert.AreEqual(3, targets.Length); ICollection <DatanodeStorageInfo> targetSet = new HashSet <DatanodeStorageInfo>(Arrays .AsList(targets)); for (int i_1 = 3; i_1 < storages.Length; i_1++) { NUnit.Framework.Assert.IsTrue(targetSet.Contains(storages[i_1])); } } finally { dataNodes[0].StopDecommission(); dataNodes[1].StopDecommission(); dataNodes[2].StopDecommission(); namenode.GetNamesystem().WriteUnlock(); } }
private static void UpdateHeartbeatWithUsage(DatanodeDescriptor dn, long capacity , long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed , int xceiverCount, int volFailures) { dn.GetStorageInfos()[0].SetUtilizationForTesting(capacity, dfsUsed, remaining, blockPoolUsed ); dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), dnCacheCapacity , dnCacheUsed, xceiverCount, volFailures, null); }
private void AddNodes(IEnumerable <DatanodeDescriptor> nodesToAdd) { NetworkTopology cluster = bm.GetDatanodeManager().GetNetworkTopology(); // construct network topology foreach (DatanodeDescriptor dn in nodesToAdd) { cluster.Add(dn); dn.GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L); dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), 0L, 0L, 0, 0, null); bm.GetDatanodeManager().CheckIfClusterIsNowMultiRack(dn); } }
public virtual void TestBlocksAreNotUnderreplicatedInSingleRack() { IList <DatanodeDescriptor> nodes = ImmutableList.Of(BlockManagerTestUtil.GetDatanodeDescriptor ("1.1.1.1", "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("2.2.2.2" , "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("3.3.3.3", "/rackA" , true), BlockManagerTestUtil.GetDatanodeDescriptor("4.4.4.4", "/rackA", true), BlockManagerTestUtil.GetDatanodeDescriptor("5.5.5.5", "/rackA", true), BlockManagerTestUtil .GetDatanodeDescriptor("6.6.6.6", "/rackA", true)); AddNodes(nodes); IList <DatanodeDescriptor> origNodes = nodes.SubList(0, 3); for (int i = 0; i < NumTestIters; i++) { DoTestSingleRackClusterIsSufficientlyReplicated(i, origNodes); } }
public static void SetupCluster() { Configuration conf = new HdfsConfiguration(); string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2", "/rack2" }; storages = DFSTestUtil.CreateDatanodeStorageInfos(racks); dataNodes = DFSTestUtil.ToDatanodeDescriptor(storages); FileSystem.SetDefaultUri(conf, "hdfs://localhost:0"); conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "0.0.0.0:0"); FilePath baseDir = PathUtils.GetTestDir(typeof(TestReplicationPolicy)); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(baseDir, "name").GetPath ()); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForWriteKey, true); conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, true); DFSTestUtil.FormatNameNode(conf); namenode = new NameNode(conf); int blockSize = 1024; dnrList = new AList <DatanodeRegistration>(); dnManager = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager(); // Register DNs for (int i = 0; i < 6; i++) { DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i], new StorageInfo (HdfsServerConstants.NodeType.DataNode), new ExportedBlockKeys(), VersionInfo.GetVersion ()); dnrList.AddItem(dnr); dnManager.RegisterDatanode(dnr); dataNodes[i].GetStorageInfos()[0].SetUtilizationForTesting(2 * HdfsConstants.MinBlocksForWrite * blockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * blockSize, 0L); dataNodes[i].UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dataNodes [i]), 0L, 0L, 0, 0, null); } }
public virtual void TestPendingAndInvalidate() { Configuration Conf = new HdfsConfiguration(); Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024); Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval); Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval ); MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount ).Build(); cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); DistributedFileSystem fs = cluster.GetFileSystem(); try { // 1. create a file Path filePath = new Path("/tmp.txt"); DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L); // 2. disable the heartbeats foreach (DataNode dn in cluster.GetDataNodes()) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true); } // 3. mark a couple of blocks as corrupt LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath .ToString(), 0, 1).Get(0); cluster.GetNamesystem().WriteLock(); try { bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID" , "TEST"); bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID" , "TEST"); } finally { cluster.GetNamesystem().WriteUnlock(); } BlockManagerTestUtil.ComputeAllPendingWork(bm); BlockManagerTestUtil.UpdateState(bm); NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L); NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock ().GetLocalBlock()), 2); // 4. delete the file fs.Delete(filePath, true); // retry at most 10 times, each time sleep for 1s. Note that 10s is much // less than the default pending record timeout (5~10min) int retries = 10; long pendingNum = bm.GetPendingReplicationBlocksCount(); while (pendingNum != 0 && retries-- > 0) { Sharpen.Thread.Sleep(1000); // let NN do the deletion BlockManagerTestUtil.UpdateState(bm); pendingNum = bm.GetPendingReplicationBlocksCount(); } NUnit.Framework.Assert.AreEqual(pendingNum, 0L); } finally { cluster.Shutdown(); } }
public virtual void TestBlockReceived() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build(); cluster.WaitActive(); DistributedFileSystem hdfs = cluster.GetFileSystem(); FSNamesystem fsn = cluster.GetNamesystem(); BlockManager blkManager = fsn.GetBlockManager(); string file = "/tmp.txt"; Path filePath = new Path(file); short replFactor = 1; DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0); // temporarily stop the heartbeat AList <DataNode> datanodes = cluster.GetDataNodes(); for (int i = 0; i < DatanodeCount; i++) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true); } hdfs.SetReplication(filePath, (short)DatanodeCount); BlockManagerTestUtil.ComputeAllPendingWork(blkManager); NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size()); INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile(); Block[] blocks = fileNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications .GetNumReplicas(blocks[0])); LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0); DatanodeInfo existingDn = (locatedBlock.GetLocations())[0]; int reportDnNum = 0; string poolId = cluster.GetNamesystem().GetBlockPoolId(); // let two datanodes (other than the one that already has the data) to // report to NN for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++) { if (!datanodes[i_1].GetDatanodeId().Equals(existingDn)) { DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId); StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, string.Empty) }) }; cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report); reportDnNum++; } } NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications .GetNumReplicas(blocks[0])); // let the same datanodes report again for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++) { if (!datanodes[i_2].GetDatanodeId().Equals(existingDn)) { DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId); StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo [] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, string.Empty) }) }; cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report); reportDnNum++; } } NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications .GetNumReplicas(blocks[0])); // re-enable heartbeat for the datanode that has data for (int i_3 = 0; i_3 < DatanodeCount; i_3++) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false); DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]); } Sharpen.Thread.Sleep(5000); NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestProcesOverReplicateBlock() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString (2)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fs = cluster.GetFileSystem(); try { Path fileName = new Path("/foo1"); DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)3); // corrupt the block on datanode 0 ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName); NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block)); MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0); // remove block scanner log to trigger block scanning FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster .GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent ()).GetParent(), "scanner.cursor"); //wait for one minute for deletion to succeed; for (int i = 0; !scanCursor.Delete(); i++) { NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath() + " in one minute", i < 60); try { Sharpen.Thread.Sleep(1000); } catch (Exception) { } } // restart the datanode so the corrupt replica will be detected cluster.RestartDataNode(dnProps); DFSTestUtil.WaitReplication(fs, fileName, (short)2); string blockPoolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[2], blockPoolId); FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager(); try { namesystem.WriteLock(); lock (hm) { // set live datanode's remaining space to be 0 // so they will be chosen to be deleted when over-replication occurs string corruptMachineName = corruptDataNode.GetXferAddr(); foreach (DatanodeDescriptor datanode in hm.GetDatanodes()) { if (!corruptMachineName.Equals(datanode.GetXferAddr())) { datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L); datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode ), 0L, 0L, 0, 0, null); } } // decrease the replication factor to 1; NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1); // corrupt one won't be chosen to be excess one // without 4910 the number of live replicas would be 0: block gets lost NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas ()); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }
public virtual void TestNodeCount() { // start a mini dfs cluster of 2 nodes Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor ).Build(); try { FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager(); FileSystem fs = cluster.GetFileSystem(); // populate the cluster with a one block file Path FilePath = new Path("/testfile"); DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L); DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, FilePath); // keep a copy of all datanode descriptor DatanodeDescriptor[] datanodes = hm.GetDatanodes(); // start two new nodes cluster.StartDataNodes(conf, 2, true, null, null); cluster.WaitActive(); // bring down first datanode DatanodeDescriptor datanode = datanodes[0]; MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(datanode.GetXferAddr ()); // make sure that NN detects that the datanode is down BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), datanode.GetXferAddr ()); // the block will be replicated DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor); // restart the first datanode cluster.RestartDataNode(dnprop); cluster.WaitActive(); // check if excessive replica is detected (transient) InitializeTimeout(Timeout); while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() == 0) { CheckTimeout("excess replicas not detected"); } // find out a non-excess node DatanodeDescriptor nonExcessDN = null; foreach (DatanodeStorageInfo storage in bm.blocksMap.GetStorages(block.GetLocalBlock ())) { DatanodeDescriptor dn = storage.GetDatanodeDescriptor(); ICollection <Block> blocks = bm.excessReplicateMap[dn.GetDatanodeUuid()]; if (blocks == null || !blocks.Contains(block.GetLocalBlock())) { nonExcessDN = dn; break; } } NUnit.Framework.Assert.IsTrue(nonExcessDN != null); // bring down non excessive datanode dnprop = cluster.StopDataNode(nonExcessDN.GetXferAddr()); // make sure that NN detects that the datanode is down BlockManagerTestUtil.NoticeDeadDatanode(cluster.GetNameNode(), nonExcessDN.GetXferAddr ()); // The block should be replicated InitializeTimeout(Timeout); while (CountNodes(block.GetLocalBlock(), namesystem).LiveReplicas() != ReplicationFactor ) { CheckTimeout("live replica count not correct", 1000); } // restart the first datanode cluster.RestartDataNode(dnprop); cluster.WaitActive(); // check if excessive replica is detected (transient) InitializeTimeout(Timeout); while (CountNodes(block.GetLocalBlock(), namesystem).ExcessReplicas() != 2) { CheckTimeout("excess replica count not equal to 2"); } } finally { cluster.Shutdown(); } }