public virtual void TestUpdateNodeResource() { UpdateNodeResourceRequest request = recordFactory.NewRecordInstance <UpdateNodeResourceRequest >(); UpdateNodeResourceResponse response = client.UpdateNodeResource(request); NUnit.Framework.Assert.IsNotNull(response); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual UpdateNodeResourceResponse UpdateNodeResource(UpdateNodeResourceRequest request) { string argName = "updateNodeResource"; UserGroupInformation user = CheckAcls(argName); CheckRMStatus(user.GetShortUserName(), argName, "update node resource."); IDictionary <NodeId, ResourceOption> nodeResourceMap = request.GetNodeResourceMap( ); ICollection <NodeId> nodeIds = nodeResourceMap.Keys; // verify nodes are all valid first. // if any invalid nodes, throw exception instead of partially updating // valid nodes. foreach (NodeId nodeId in nodeIds) { RMNode node = this.rmContext.GetRMNodes()[nodeId]; if (node == null) { Log.Error("Resource update get failed on all nodes due to change " + "resource on an unrecognized node: " + nodeId); throw RPCUtil.GetRemoteException("Resource update get failed on all nodes due to change resource " + "on an unrecognized node: " + nodeId); } } // do resource update on each node. // Notice: it is still possible to have invalid NodeIDs as nodes decommission // may happen just at the same time. This time, only log and skip absent // nodes without throwing any exceptions. bool allSuccess = true; foreach (KeyValuePair <NodeId, ResourceOption> entry in nodeResourceMap) { ResourceOption newResourceOption = entry.Value; NodeId nodeId_1 = entry.Key; RMNode node = this.rmContext.GetRMNodes()[nodeId_1]; if (node == null) { Log.Warn("Resource update get failed on an unrecognized node: " + nodeId_1); allSuccess = false; } else { // update resource to RMNode this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeResourceUpdateEvent (nodeId_1, newResourceOption)); Log.Info("Update resource on node(" + node.GetNodeID() + ") with resource(" + newResourceOption .ToString() + ")"); } } if (allSuccess) { RMAuditLogger.LogSuccess(user.GetShortUserName(), argName, "AdminService"); } UpdateNodeResourceResponse response = UpdateNodeResourceResponse.NewInstance(); return(response); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual UpdateNodeResourceResponse UpdateNodeResource(UpdateNodeResourceRequest request) { YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto requestProto = ((UpdateNodeResourceRequestPBImpl)request).GetProto(); try { return(new UpdateNodeResourceResponsePBImpl(proxy.UpdateNodeResource(null, requestProto ))); } catch (ServiceException e) { RPCUtil.UnwrapAndThrowException(e); return(null); } }
public virtual void TestResourceOverCommit() { MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 4 * Gb); RMApp app1 = rm.SubmitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); // check node report, 2 GB used and 2 GB available NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // add request for containers am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 2 * Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } IList <Container> allocated1 = alloc1Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated1.Count); NUnit.Framework.Assert.AreEqual(2 * Gb, allocated1[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId()); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); // check node report, 4 GB used and 0 GB available NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); // check container is assigned with 2 GB. Container c1 = allocated1[0]; NUnit.Framework.Assert.AreEqual(2 * Gb, c1.GetResource().GetMemory()); // update node resource to 2 GB, so resource is over-consumed. IDictionary <NodeId, ResourceOption> nodeResourceMap = new Dictionary <NodeId, ResourceOption >(); nodeResourceMap[nm1.GetNodeId()] = ResourceOption.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2 * Gb, 1), -1); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.NewInstance(nodeResourceMap ); AdminService @as = rm.adminService; @as.UpdateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(-2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState .Complete, string.Empty, 0); nm1.ContainerStatus(containerStatus); int waitCount = 0; while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20) { Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count); NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses( ).Count); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); // As container return 2 GB back, the available resource becomes 0 again. NUnit.Framework.Assert.AreEqual(0 * Gb, report_nm1.GetAvailableResource().GetMemory ()); rm.Stop(); }