public virtual void TestNodeRegistrationVersionLessThanRM() { WriteToHostsFile("host2"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM"); rm = new MockRM(conf); rm.Start(); string nmVersion = "1.9.9"; ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = NodeId.NewInstance("host2", 1234); Resource capability = BuilderUtils.NewResource(1024, 1); req.SetResource(capability); req.SetNodeId(nodeId); req.SetHttpPort(1234); req.SetNMVersion(nmVersion); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction()); NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager " + "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage ().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version " )); }
public MockNM(string nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker , string version) { // scale vcores based on the requested memory this.memory = memory; this.vCores = vcores; this.resourceTracker = resourceTracker; this.version = version; string[] splits = nodeIdStr.Split(":"); nodeId = BuilderUtils.NewNodeId(splits[0], System.Convert.ToInt32(splits[1])); }
public virtual void TestNodeRegistrationWithMinimumAllocations() { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationMb, "2048"); conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationVcores, "4"); rm = new MockRM(conf); rm.Start(); ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = BuilderUtils.NewNodeId("host", 1234); req.SetNodeId(nodeId); Resource capability = BuilderUtils.NewResource(1024, 1); req.SetResource(capability); RegisterNodeManagerResponse response1 = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response1.GetNodeAction()); capability.SetMemory(2048); capability.SetVirtualCores(1); req.SetResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response2.GetNodeAction()); capability.SetMemory(1024); capability.SetVirtualCores(4); req.SetResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response3.GetNodeAction()); capability.SetMemory(2048); capability.SetVirtualCores(4); req.SetResource(capability); RegisterNodeManagerResponse response4 = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response4.GetNodeAction()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public NodeManager(string hostName, int containerManagerPort, int httpPort, string rackName, Resource capability, ResourceManager resourceManager) { this.containerManagerAddress = hostName + ":" + containerManagerPort; this.nodeHttpAddress = hostName + ":" + httpPort; this.rackName = rackName; this.resourceTrackerService = resourceManager.GetResourceTrackerService(); this.capability = capability; Resources.AddTo(available, capability); this.nodeId = NodeId.NewInstance(hostName, containerManagerPort); RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest >(); request.SetHttpPort(httpPort); request.SetResource(capability); request.SetNodeId(this.nodeId); request.SetNMVersion(YarnVersionInfo.GetVersion()); resourceTrackerService.RegisterNodeManager(request); this.resourceManager = resourceManager; resourceManager.GetResourceScheduler().GetNodeReport(this.nodeId); }
public virtual void TestNodeRegistrationFailure() { WriteToHostsFile("host1"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); rm = new MockRM(conf); rm.Start(); ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = NodeId.NewInstance("host2", 1234); req.SetNodeId(nodeId); req.SetHttpPort(1234); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction()); NUnit.Framework.Assert.AreEqual("Disallowed NodeManager from host2, Sending SHUTDOWN signal to the NodeManager." , response.GetDiagnosticsMessage()); }
public virtual void TestNodeRegistrationSuccess() { WriteToHostsFile("host2"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); rm = new MockRM(conf); rm.Start(); ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = NodeId.NewInstance("host2", 1234); Resource capability = BuilderUtils.NewResource(1024, 1); req.SetResource(capability); req.SetNodeId(nodeId); req.SetHttpPort(1234); req.SetNMVersion(YarnVersionInfo.GetVersion()); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response.GetNodeAction()); }
internal virtual void SetResourceTrackerService(ResourceTrackerService resourceTrackerService ) { activeServiceContext.SetResourceTrackerService(resourceTrackerService); }
public virtual void SetResourceTrackerService(ResourceTrackerService resourceTracker ) { this.resourceTracker = resourceTracker; }
public MockNM(string nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker ) : this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.GetVersion()) { }
public MockNM(string nodeIdStr, int memory, ResourceTrackerService resourceTracker ) : this(nodeIdStr, memory, Math.Max(1, (memory * YarnConfiguration.DefaultNmVcores ) / YarnConfiguration.DefaultNmPmemMb), resourceTracker) { }
internal virtual void SetResourceTrackerService(ResourceTrackerService resourceTrackerService ) { this.resourceTrackerService = resourceTrackerService; }