public virtual void TestDecommissionWithExcludeHosts() { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath()); WriteToHostsFile(string.Empty); DrainDispatcher dispatcher = new DrainDispatcher(); rm = new _MockRM_162(dispatcher, conf); rm.Start(); MockNM nm1 = rm.RegisterNode("host1:1234", 5120); MockNM nm2 = rm.RegisterNode("host2:5678", 10240); MockNM nm3 = rm.RegisterNode("localhost:4433", 1024); dispatcher.Await(); int metricCount = ClusterMetrics.GetMetrics().GetNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); dispatcher.Await(); // To test that IPs also work string ip = NetUtils.NormalizeHostName("localhost"); WriteToHostsFile("host2", ip); rm.GetNodesListManager().RefreshNodes(conf); CheckDecommissionedNMCount(rm, metricCount + 2); nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction .Shutdown.Equals(nodeHeartbeat.GetNodeAction())); nodeHeartbeat = nm3.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction .Shutdown.Equals(nodeHeartbeat.GetNodeAction())); dispatcher.Await(); WriteToHostsFile(string.Empty); rm.GetNodesListManager().RefreshNodes(conf); nm3 = rm.RegisterNode("localhost:4433", 1024); dispatcher.Await(); nodeHeartbeat = nm3.NodeHeartbeat(true); dispatcher.Await(); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); // decommissined node is 1 since 1 node is rejoined after updating exclude // file CheckDecommissionedNMCount(rm, metricCount + 1); }
public virtual void TestAddNewExcludePathToConfiguration() { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("host1:1234", 5120); MockNM nm2 = rm.RegisterNode("host2:5678", 10240); ClusterMetrics metrics = ClusterMetrics.GetMetrics(); System.Diagnostics.Debug.Assert((metrics != null)); int initialMetricCount = metrics.GetNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction()); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction()); WriteToHostsFile("host2"); conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath()); rm.GetNodesListManager().RefreshNodes(conf); CheckDecommissionedNMCount(rm, ++initialMetricCount); nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual("Node should not have been decomissioned.", NodeAction .Normal, nodeHeartbeat.GetNodeAction()); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual("Node should have been decomissioned but is in state" + nodeHeartbeat.GetNodeAction(), NodeAction.Shutdown, nodeHeartbeat.GetNodeAction ()); }
public virtual void TestDecommissionWithIncludeHosts() { WriteToHostsFile("localhost", "host1", "host2"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("host1:1234", 5120); MockNM nm2 = rm.RegisterNode("host2:5678", 10240); MockNM nm3 = rm.RegisterNode("localhost:4433", 1024); ClusterMetrics metrics = ClusterMetrics.GetMetrics(); System.Diagnostics.Debug.Assert((metrics != null)); int metricCount = metrics.GetNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm3.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); // To test that IPs also work string ip = NetUtils.NormalizeHostName("localhost"); WriteToHostsFile("host1", ip); rm.GetNodesListManager().RefreshNodes(conf); CheckDecommissionedNMCount(rm, ++metricCount); nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); NUnit.Framework.Assert.AreEqual(1, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs ()); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue("Node is not decommisioned.", NodeAction.Shutdown.Equals (nodeHeartbeat.GetNodeAction())); nodeHeartbeat = nm3.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); NUnit.Framework.Assert.AreEqual(metricCount, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs ()); }
public virtual void TestRefreshNodesWithFileSystemBasedConfigurationProvider() { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); // upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } FilePath excludeHostsFile = new FilePath(tmpDir.ToString(), "excludeHosts"); if (excludeHostsFile.Exists()) { excludeHostsFile.Delete(); } if (!excludeHostsFile.CreateNewFile()) { NUnit.Framework.Assert.Fail("Can not create " + "excludeHosts"); } PrintWriter fileWriter = new PrintWriter(excludeHostsFile); fileWriter.Write("0.0.0.0:123"); fileWriter.Close(); UploadToRemoteFileSystem(new Path(excludeHostsFile.GetAbsolutePath())); Configuration yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.RmNodesExcludeFilePath, this.workingPath + "/excludeHosts" ); UploadConfiguration(yarnConf, YarnConfiguration.YarnSiteConfigurationFile); rm.adminService.RefreshNodes(RefreshNodesRequest.NewInstance()); ICollection <string> excludeHosts = rm.GetNodesListManager().GetHostsReader().GetExcludedHosts (); NUnit.Framework.Assert.IsTrue(excludeHosts.Count == 1); NUnit.Framework.Assert.IsTrue(excludeHosts.Contains("0.0.0.0:123")); }