public virtual void VerifyClusterGeneric(long clusterid, long startedon, string state , string haState, string haZooKeeperConnectionState, string hadoopVersionBuiltOn , string hadoopBuildVersion, string hadoopVersion, string resourceManagerVersionBuiltOn , string resourceManagerBuildVersion, string resourceManagerVersion) { NUnit.Framework.Assert.AreEqual("clusterId doesn't match: ", ResourceManager.GetClusterTimeStamp (), clusterid); NUnit.Framework.Assert.AreEqual("startedOn doesn't match: ", ResourceManager.GetClusterTimeStamp (), startedon); NUnit.Framework.Assert.IsTrue("stated doesn't match: " + state, state.Matches(Service.STATE .Inited.ToString())); NUnit.Framework.Assert.IsTrue("HA state doesn't match: " + haState, haState.Matches ("INITIALIZING")); WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate (), hadoopVersionBuiltOn); WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion (), hadoopBuildVersion); WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(), hadoopVersion); WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo .GetDate(), resourceManagerVersionBuiltOn); WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo .GetBuildVersion(), resourceManagerBuildVersion); WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion (), resourceManagerVersion); }
public ClusterInfo(ResourceManager rm) { // JAXB needs this long ts = ResourceManager.GetClusterTimeStamp(); this.id = ts; this.state = rm.GetServiceState(); this.haState = rm.GetRMContext().GetHAServiceState(); this.rmStateStoreName = rm.GetRMContext().GetStateStore().GetType().FullName; this.startedOn = ts; this.resourceManagerVersion = YarnVersionInfo.GetVersion(); this.resourceManagerBuildVersion = YarnVersionInfo.GetBuildVersion(); this.resourceManagerVersionBuiltOn = YarnVersionInfo.GetDate(); this.hadoopVersion = VersionInfo.GetVersion(); this.hadoopBuildVersion = VersionInfo.GetBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.GetDate(); this.haZooKeeperConnectionState = rm.GetRMContext().GetRMAdminService().GetHAZookeeperConnectionState (); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public NodeManager(string hostName, int containerManagerPort, int httpPort, string rackName, Resource capability, ResourceManager resourceManager) { this.containerManagerAddress = hostName + ":" + containerManagerPort; this.nodeHttpAddress = hostName + ":" + httpPort; this.rackName = rackName; this.resourceTrackerService = resourceManager.GetResourceTrackerService(); this.capability = capability; Resources.AddTo(available, capability); this.nodeId = NodeId.NewInstance(hostName, containerManagerPort); RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest >(); request.SetHttpPort(httpPort); request.SetResource(capability); request.SetNodeId(this.nodeId); request.SetNMVersion(YarnVersionInfo.GetVersion()); resourceTrackerService.RegisterNodeManager(request); this.resourceManager = resourceManager; resourceManager.GetResourceScheduler().GetNodeReport(this.nodeId); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.GetNodeId(); this.httpPort = this.context.GetHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.GetVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = GetRMClient(); RegisterWithRM(); base.ServiceStart(); StartStatusUpdater(); } catch (Exception e) { string errorMessage = "Unexpected error starting NodeStatusUpdater"; Log.Error(errorMessage, e); throw new YarnRuntimeException(e); } }
public NodeInfo(Context context, ResourceView resourceView) { // JAXB needs this this.id = context.GetNodeId().ToString(); this.nodeHostName = context.GetNodeId().GetHost(); this.totalVmemAllocatedContainersMB = resourceView.GetVmemAllocatedForContainers( ) / BytesInMb; this.vmemCheckEnabled = resourceView.IsVmemCheckEnabled(); this.totalPmemAllocatedContainersMB = resourceView.GetPmemAllocatedForContainers( ) / BytesInMb; this.pmemCheckEnabled = resourceView.IsPmemCheckEnabled(); this.totalVCoresAllocatedContainers = resourceView.GetVCoresAllocatedForContainers (); this.nodeHealthy = context.GetNodeHealthStatus().GetIsNodeHealthy(); this.lastNodeUpdateTime = context.GetNodeHealthStatus().GetLastHealthReportTime(); this.healthReport = context.GetNodeHealthStatus().GetHealthReport(); this.nodeManagerVersion = YarnVersionInfo.GetVersion(); this.nodeManagerBuildVersion = YarnVersionInfo.GetBuildVersion(); this.nodeManagerVersionBuiltOn = YarnVersionInfo.GetDate(); this.hadoopVersion = VersionInfo.GetVersion(); this.hadoopBuildVersion = VersionInfo.GetBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.GetDate(); }
public virtual void VerifyNodeInfoGeneric(string id, string healthReport, long totalVmemAllocatedContainersMB , long totalPmemAllocatedContainersMB, long totalVCoresAllocatedContainers, bool vmemCheckEnabled, bool pmemCheckEnabled, long lastNodeUpdateTime, bool nodeHealthy , string nodeHostName, string hadoopVersionBuiltOn, string hadoopBuildVersion, string hadoopVersion, string resourceManagerVersionBuiltOn, string resourceManagerBuildVersion , string resourceManagerVersion) { WebServicesTestUtils.CheckStringMatch("id", "testhost.foo.com:8042", id); WebServicesTestUtils.CheckStringMatch("healthReport", "Healthy", healthReport); NUnit.Framework.Assert.AreEqual("totalVmemAllocatedContainersMB incorrect", 15872 , totalVmemAllocatedContainersMB); NUnit.Framework.Assert.AreEqual("totalPmemAllocatedContainersMB incorrect", 16384 , totalPmemAllocatedContainersMB); NUnit.Framework.Assert.AreEqual("totalVCoresAllocatedContainers incorrect", 4000, totalVCoresAllocatedContainers); NUnit.Framework.Assert.AreEqual("vmemCheckEnabled incorrect", true, vmemCheckEnabled ); NUnit.Framework.Assert.AreEqual("pmemCheckEnabled incorrect", true, pmemCheckEnabled ); NUnit.Framework.Assert.IsTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext.GetNodeHealthStatus().GetLastHealthReportTime()); NUnit.Framework.Assert.IsTrue("nodeHealthy isn't true", nodeHealthy); WebServicesTestUtils.CheckStringMatch("nodeHostName", "testhost.foo.com", nodeHostName ); WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate (), hadoopVersionBuiltOn); WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion (), hadoopBuildVersion); WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(), hadoopVersion); WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo .GetDate(), resourceManagerVersionBuiltOn); WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo .GetBuildVersion(), resourceManagerBuildVersion); WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion (), resourceManagerVersion); }
public virtual void TestNodeRegistrationSuccess() { WriteToHostsFile("host2"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); rm = new MockRM(conf); rm.Start(); ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = NodeId.NewInstance("host2", 1234); Resource capability = BuilderUtils.NewResource(1024, 1); req.SetResource(capability); req.SetNodeId(nodeId); req.SetHttpPort(1234); req.SetNMVersion(YarnVersionInfo.GetVersion()); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response.GetNodeAction()); }
/// <exception cref="System.Exception"/> public virtual void TestResourceTrackerOnHA() { NodeId nodeId = NodeId.NewInstance("localhost", 0); Resource resource = Resource.NewInstance(2048, 4); // make sure registerNodeManager works when failover happens RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(nodeId , 0, resource, YarnVersionInfo.GetVersion(), null, null); resourceTracker.RegisterNodeManager(request); NUnit.Framework.Assert.IsTrue(WaitForNodeManagerToConnect(10000, nodeId)); // restart the failover thread, and make sure nodeHeartbeat works failoverThread = CreateAndStartFailoverThread(); NodeStatus status = NodeStatus.NewInstance(NodeId.NewInstance("localhost", 0), 0, null, null, null); NodeHeartbeatRequest request2 = NodeHeartbeatRequest.NewInstance(status, null, null ); resourceTracker.NodeHeartbeat(request2); }
public MockNM(string nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker ) : this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.GetVersion()) { }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest request) { NodeId nodeId = request.GetNodeId(); string host = nodeId.GetHost(); int cmPort = nodeId.GetPort(); int httpPort = request.GetHttpPort(); Resource capability = request.GetResource(); string nodeManagerVersion = request.GetNMVersion(); RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse >(); if (!minimumNodeManagerVersion.Equals("NONE")) { if (minimumNodeManagerVersion.Equals("EqualToRM")) { minimumNodeManagerVersion = YarnVersionInfo.GetVersion(); } if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion , minimumNodeManagerVersion)) < 0) { string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version " + minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } } // Check if this node is a 'valid' node if (!this.nodesListManager.IsValidNode(host)) { string message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } // Check if this node has minimum allocations if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores) { string message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + " signal to the NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey()); response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey()); RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve (host), capability, nodeManagerVersion); RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode); if (oldNode == null) { this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId , request.GetNMContainerStatuses(), request.GetRunningApplications())); } else { Log.Info("Reconnect from the node at: " + host); this.nmLivelinessMonitor.Unregister(nodeId); // Reset heartbeat ID since node just restarted. oldNode.ResetLastNodeHeartBeatResponse(); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent( nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses ())); } // On every node manager register we will be clearing NMToken keys if // present for any running application. this.nmTokenSecretManager.RemoveNodeKey(nodeId); this.nmLivelinessMonitor.Register(nodeId); // Handle received container status, this should be processed after new // RMNode inserted if (!rmContext.IsWorkPreservingRecoveryEnabled()) { if (!request.GetNMContainerStatuses().IsEmpty()) { Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses ()); foreach (NMContainerStatus status in request.GetNMContainerStatuses()) { HandleNMContainerStatus(status, nodeId); } } } string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: " + httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId " + nodeId; Log.Info(message_1); response.SetNodeAction(NodeAction.Normal); response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp()); response.SetRMVersion(YarnVersionInfo.GetVersion()); return(response); }
/// <exception cref="System.Exception"/> public virtual MockNM RegisterNode(string nodeIdStr, int memory, int vCores, IList <ApplicationId> runningApplications) { MockNM nm = new MockNM(nodeIdStr, memory, vCores, GetResourceTrackerService(), YarnVersionInfo .GetVersion()); nm.RegisterNode(runningApplications); return(nm); }