internal virtual IList <Block> InvalidateWork(DatanodeDescriptor dn) { lock (this) { long delay = GetInvalidationDelay(); if (delay > 0) { if (BlockManager.Log.IsDebugEnabled()) { BlockManager.Log.Debug("Block deletion is delayed during NameNode startup. " + "The deletion will start after " + delay + " ms."); } return(null); } LightWeightHashSet <Block> set = node2blocks[dn]; if (set == null) { return(null); } // # blocks that can be sent in one message is limited int limit = blockInvalidateLimit; IList <Block> toInvalidate = set.PollN(limit); // If we send everything in this message, remove this node entry if (set.IsEmpty()) { Remove(dn); } dn.AddBlocksToBeInvalidated(toInvalidate); numBlocks -= toInvalidate.Count; return(toInvalidate); } }
public virtual void TestGetInvalidateBlocks() { int MaxBlocks = 10; int RemainingBlocks = 2; int MaxLimit = MaxBlocks - RemainingBlocks; DatanodeDescriptor dd = DFSTestUtil.GetLocalDatanodeDescriptor(); AList <Block> blockList = new AList <Block>(MaxBlocks); for (int i = 0; i < MaxBlocks; i++) { blockList.AddItem(new Block(i, 0, GenerationStamp.LastReservedStamp)); } dd.AddBlocksToBeInvalidated(blockList); Block[] bc = dd.GetInvalidateBlocks(MaxLimit); NUnit.Framework.Assert.AreEqual(bc.Length, MaxLimit); bc = dd.GetInvalidateBlocks(MaxLimit); NUnit.Framework.Assert.AreEqual(bc.Length, RemainingBlocks); }
public virtual void TestHeartbeat() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager (); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg); string storageID = DatanodeStorage.GenerateUuid(); dd.UpdateStorage(new DatanodeStorage(storageID)); int RemainingBlocks = 1; int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey , 2); int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault; int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks; int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks; DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID ) }; try { namesystem.WriteLock(); lock (hm) { for (int i = 0; i < MaxReplicateBlocks; i++) { dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget ); } DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands (); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks); for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++) { blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp)); } dd.AddBlocksToBeInvalidated(blockList); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction ()); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(0, cmds.Length); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }