/// <summary>Implements getLiveNodeManagers()</summary> /// <returns>JSON formatted string containing statuses of all node managers</returns> public virtual string GetLiveNodeManagers() { // RMNMInfoBeans ICollection <RMNode> nodes = this.rmContext.GetRMNodes().Values; IList <RMNMInfo.InfoMap> nodesInfo = new AList <RMNMInfo.InfoMap>(); foreach (RMNode ni in nodes) { SchedulerNodeReport report = scheduler.GetNodeReport(ni.GetNodeID()); RMNMInfo.InfoMap info = new RMNMInfo.InfoMap(); info["HostName"] = ni.GetHostName(); info["Rack"] = ni.GetRackName(); info["State"] = ni.GetState().ToString(); info["NodeId"] = ni.GetNodeID(); info["NodeHTTPAddress"] = ni.GetHttpAddress(); info["LastHealthUpdate"] = ni.GetLastHealthReportTime(); info["HealthReport"] = ni.GetHealthReport(); info["NodeManagerVersion"] = ni.GetNodeManagerVersion(); if (report != null) { info["NumContainers"] = report.GetNumContainers(); info["UsedMemoryMB"] = report.GetUsedResource().GetMemory(); info["AvailableMemoryMB"] = report.GetAvailableResource().GetMemory(); } nodesInfo.AddItem(info); } return(JSON.ToString(nodesInfo)); }
public NodeInfo(RMNode ni, ResourceScheduler sched) { // JAXB needs this NodeId id = ni.GetNodeID(); SchedulerNodeReport report = sched.GetNodeReport(id); this.numContainers = 0; this.usedMemoryMB = 0; this.availMemoryMB = 0; if (report != null) { this.numContainers = report.GetNumContainers(); this.usedMemoryMB = report.GetUsedResource().GetMemory(); this.availMemoryMB = report.GetAvailableResource().GetMemory(); this.usedVirtualCores = report.GetUsedResource().GetVirtualCores(); this.availableVirtualCores = report.GetAvailableResource().GetVirtualCores(); } this.id = id.ToString(); this.rack = ni.GetRackName(); this.nodeHostName = ni.GetHostName(); this.state = ni.GetState(); this.nodeHTTPAddress = ni.GetHttpAddress(); this.lastHealthUpdate = ni.GetLastHealthReportTime(); this.healthReport = ni.GetHealthReport().ToString(); this.version = ni.GetNodeManagerVersion(); // add labels ICollection <string> labelSet = ni.GetNodeLabels(); if (labelSet != null) { Sharpen.Collections.AddAll(nodeLabels, labelSet); nodeLabels.Sort(); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeInfoGeneric(MockNM nm, string state, string rack, string id, string nodeHostName, string nodeHTTPAddress, long lastHealthUpdate, string healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores , long availVirtualCores, string version) { RMNode node = rm.GetRMContext().GetRMNodes()[nm.GetNodeId()]; ResourceScheduler sched = rm.GetResourceScheduler(); SchedulerNodeReport report = sched.GetNodeReport(nm.GetNodeId()); WebServicesTestUtils.CheckStringMatch("state", node.GetState().ToString(), state); WebServicesTestUtils.CheckStringMatch("rack", node.GetRackName(), rack); WebServicesTestUtils.CheckStringMatch("id", nm.GetNodeId().ToString(), id); WebServicesTestUtils.CheckStringMatch("nodeHostName", nm.GetNodeId().GetHost(), nodeHostName ); WebServicesTestUtils.CheckStringMatch("healthReport", node.GetHealthReport().ToString (), healthReport); string expectedHttpAddress = nm.GetNodeId().GetHost() + ":" + nm.GetHttpPort(); WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress ); WebServicesTestUtils.CheckStringMatch("version", node.GetNodeManagerVersion(), version ); long expectedHealthUpdate = node.GetLastHealthReportTime(); NUnit.Framework.Assert.AreEqual("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate); if (report != null) { NUnit.Framework.Assert.AreEqual("numContainers doesn't match: " + numContainers, report.GetNumContainers(), numContainers); NUnit.Framework.Assert.AreEqual("usedMemoryMB doesn't match: " + usedMemoryMB, report .GetUsedResource().GetMemory(), usedMemoryMB); NUnit.Framework.Assert.AreEqual("availMemoryMB doesn't match: " + availMemoryMB, report.GetAvailableResource().GetMemory(), availMemoryMB); NUnit.Framework.Assert.AreEqual("usedVirtualCores doesn't match: " + usedVirtualCores , report.GetUsedResource().GetVirtualCores(), usedVirtualCores); NUnit.Framework.Assert.AreEqual("availVirtualCores doesn't match: " + availVirtualCores , report.GetAvailableResource().GetVirtualCores(), availVirtualCores); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }