public virtual void TestStorageWithRemainingCapacity() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = FileSystem.Get(conf); Path file1 = null; try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg); // By default, MiniDFSCluster will create 1 datanode with 2 storages. // Assigning 64k for remaining storage capacity and will //create a file with 100k. foreach (DatanodeStorageInfo storage in dd.GetStorageInfos()) { storage.SetUtilizationForTesting(65536, 0, 65536, 0); } //sum of the remaining capacity of both the storages dd.SetRemaining(131072); file1 = new Path("testRemainingStorage.dat"); try { DFSTestUtil.CreateFile(fs, file1, 102400, 102400, 102400, (short)1, unchecked ((int )(0x1BAD5EED))); } catch (RemoteException re) { GenericTestUtils.AssertExceptionContains("nodes instead of " + "minReplication", re); } } finally { // Clean up NUnit.Framework.Assert.IsTrue(fs.Exists(file1)); fs.Delete(file1, true); NUnit.Framework.Assert.IsTrue(!fs.Exists(file1)); cluster.Shutdown(); } }
public virtual void TestHeartbeatBlockRecovery() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager (); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg1 = DataNodeTestUtils.GetDNRegistrationForBP(cluster. GetDataNodes()[0], poolId); DatanodeDescriptor dd1 = NameNodeAdapter.GetDatanode(namesystem, nodeReg1); dd1.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid())); DatanodeRegistration nodeReg2 = DataNodeTestUtils.GetDNRegistrationForBP(cluster. GetDataNodes()[1], poolId); DatanodeDescriptor dd2 = NameNodeAdapter.GetDatanode(namesystem, nodeReg2); dd2.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid())); DatanodeRegistration nodeReg3 = DataNodeTestUtils.GetDNRegistrationForBP(cluster. GetDataNodes()[2], poolId); DatanodeDescriptor dd3 = NameNodeAdapter.GetDatanode(namesystem, nodeReg3); dd3.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid())); try { namesystem.WriteLock(); lock (hm) { NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem); NameNodeAdapter.SendHeartBeat(nodeReg2, dd2, namesystem); NameNodeAdapter.SendHeartBeat(nodeReg3, dd3, namesystem); // Test with all alive nodes. DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0); DFSTestUtil.ResetLastUpdatesWithOffset(dd2, 0); DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0); DatanodeStorageInfo[] storages = new DatanodeStorageInfo[] { dd1.GetStorageInfos( )[0], dd2.GetStorageInfos()[0], dd3.GetStorageInfos()[0] }; BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction (new Block(0, 0, GenerationStamp.LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState .UnderRecovery, storages); dd1.AddBlockToBeRecovered(blockInfo); DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem) .GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction ()); BlockRecoveryCommand recoveryCommand = (BlockRecoveryCommand)cmds[0]; NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count); DatanodeInfo[] recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks (), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations(); NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length); NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1); NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2); NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3); // Test with one stale node. DFSTestUtil.ResetLastUpdatesWithOffset(dd1, 0); // More than the default stale interval of 30 seconds. DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000); DFSTestUtil.ResetLastUpdatesWithOffset(dd3, 0); blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp .LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages ); dd1.AddBlockToBeRecovered(blockInfo); cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction ()); recoveryCommand = (BlockRecoveryCommand)cmds[0]; NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count); recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks (), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations(); NUnit.Framework.Assert.AreEqual(2, recoveringNodes.Length); // dd2 is skipped. NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1); NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd3); // Test with all stale node. DFSTestUtil.ResetLastUpdatesWithOffset(dd1, -60 * 1000); // More than the default stale interval of 30 seconds. DFSTestUtil.ResetLastUpdatesWithOffset(dd2, -40 * 1000); DFSTestUtil.ResetLastUpdatesWithOffset(dd3, -80 * 1000); blockInfo = new BlockInfoContiguousUnderConstruction(new Block(0, 0, GenerationStamp .LastReservedStamp), (short)3, HdfsServerConstants.BlockUCState.UnderRecovery, storages ); dd1.AddBlockToBeRecovered(blockInfo); cmds = NameNodeAdapter.SendHeartBeat(nodeReg1, dd1, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaRecoverblock, cmds[0].GetAction ()); recoveryCommand = (BlockRecoveryCommand)cmds[0]; NUnit.Framework.Assert.AreEqual(1, recoveryCommand.GetRecoveringBlocks().Count); recoveringNodes = Sharpen.Collections.ToArray(recoveryCommand.GetRecoveringBlocks (), new BlockRecoveryCommand.RecoveringBlock[0])[0].GetLocations(); // Only dd1 is included since it heart beated and hence its not stale // when the list of recovery blocks is constructed. NUnit.Framework.Assert.AreEqual(3, recoveringNodes.Length); NUnit.Framework.Assert.AreEqual(recoveringNodes[0], dd1); NUnit.Framework.Assert.AreEqual(recoveringNodes[1], dd2); NUnit.Framework.Assert.AreEqual(recoveringNodes[2], dd3); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }
public virtual void TestHeartbeat() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); HeartbeatManager hm = namesystem.GetBlockManager().GetDatanodeManager().GetHeartbeatManager (); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg); string storageID = DatanodeStorage.GenerateUuid(); dd.UpdateStorage(new DatanodeStorage(storageID)); int RemainingBlocks = 1; int MaxReplicateLimit = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationMaxStreamsKey , 2); int MaxInvalidateLimit = DFSConfigKeys.DfsBlockInvalidateLimitDefault; int MaxInvalidateBlocks = 2 * MaxInvalidateLimit + RemainingBlocks; int MaxReplicateBlocks = 2 * MaxReplicateLimit + RemainingBlocks; DatanodeStorageInfo[] OneTarget = new DatanodeStorageInfo[] { dd.GetStorageInfo(storageID ) }; try { namesystem.WriteLock(); lock (hm) { for (int i = 0; i < MaxReplicateBlocks; i++) { dd.AddBlockToBeReplicated(new Block(i, 0, GenerationStamp.LastReservedStamp), OneTarget ); } DatanodeCommand[] cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands (); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); AList <Block> blockList = new AList <Block>(MaxInvalidateBlocks); for (int i_1 = 0; i_1 < MaxInvalidateBlocks; i_1++) { blockList.AddItem(new Block(i_1, 0, GenerationStamp.LastReservedStamp)); } dd.AddBlocksToBeInvalidated(blockList); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(MaxReplicateLimit, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(2, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaTransfer, cmds[0].GetAction() ); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[1].GetAction ()); NUnit.Framework.Assert.AreEqual(MaxInvalidateLimit, ((BlockCommand)cmds[1]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(1, cmds.Length); NUnit.Framework.Assert.AreEqual(DatanodeProtocol.DnaInvalidate, cmds[0].GetAction ()); NUnit.Framework.Assert.AreEqual(RemainingBlocks, ((BlockCommand)cmds[0]).GetBlocks ().Length); cmds = NameNodeAdapter.SendHeartBeat(nodeReg, dd, namesystem).GetCommands(); NUnit.Framework.Assert.AreEqual(0, cmds.Length); } } finally { namesystem.WriteUnlock(); } } finally { cluster.Shutdown(); } }
public virtual void TestDeletedBlockWhenAddBlockIsInEdit() { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(1).Build(); DFSClient client = null; try { cluster.WaitActive(); NUnit.Framework.Assert.AreEqual("Number of namenodes is not 2", 2, cluster.GetNumNameNodes ()); // Transitioning the namenode 0 to active. cluster.TransitionToActive(0); NUnit.Framework.Assert.IsTrue("Namenode 0 should be in active state", cluster.GetNameNode (0).IsActiveState()); NUnit.Framework.Assert.IsTrue("Namenode 1 should be in standby state", cluster.GetNameNode (1).IsStandbyState()); // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover // to true. DataNodeTestUtils.TriggerHeartbeat(cluster.GetDataNodes()[0]); FileSystem fs = cluster.GetFileSystem(0); // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale // to false. cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental (false).Build()); Path fileName = new Path("/tmp.txt"); // create a file with one block DFSTestUtil.CreateFile(fs, fileName, 10L, (short)1, 1234L); DFSTestUtil.WaitReplication(fs, fileName, (short)1); client = new DFSClient(cluster.GetFileSystem(0).GetUri(), conf); IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt" , 0, 10L).GetLocatedBlocks(); NUnit.Framework.Assert.IsTrue(locatedBlocks.Count == 1); NUnit.Framework.Assert.IsTrue(locatedBlocks[0].GetLocations().Length == 1); // add a second datanode to the cluster cluster.StartDataNodes(conf, 1, true, null, null, null, null); NUnit.Framework.Assert.AreEqual("Number of datanodes should be 2", 2, cluster.GetDataNodes ().Count); DataNode dn0 = cluster.GetDataNodes()[0]; DataNode dn1 = cluster.GetDataNodes()[1]; string activeNNBPId = cluster.GetNamesystem(0).GetBlockPoolId(); DatanodeDescriptor sourceDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem (0), dn0.GetDNRegistrationForBP(activeNNBPId)); DatanodeDescriptor destDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem (0), dn1.GetDNRegistrationForBP(activeNNBPId)); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName); Log.Info("replaceBlock: " + ReplaceBlock(block, (DatanodeInfo)sourceDnDesc, (DatanodeInfo )sourceDnDesc, (DatanodeInfo)destDnDesc)); // Waiting for the FsDatasetAsyncDsikService to delete the block Sharpen.Thread.Sleep(3000); // Triggering the incremental block report to report the deleted block to // namnemode cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental (true).Build()); cluster.TransitionToStandby(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.IsTrue("Namenode 1 should be in active state", cluster.GetNameNode (1).IsActiveState()); NUnit.Framework.Assert.IsTrue("Namenode 0 should be in standby state", cluster.GetNameNode (0).IsStandbyState()); client.Close(); // Opening a new client for new active namenode client = new DFSClient(cluster.GetFileSystem(1).GetUri(), conf); IList <LocatedBlock> locatedBlocks1 = client.GetNamenode().GetBlockLocations("/tmp.txt" , 0, 10L).GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, locatedBlocks1.Count); NUnit.Framework.Assert.AreEqual("The block should be only on 1 datanode ", 1, locatedBlocks1 [0].GetLocations().Length); } finally { IOUtils.Cleanup(null, client); cluster.Shutdown(); } }