public NodeInfo(RMNode ni, ResourceScheduler sched) { // JAXB needs this NodeId id = ni.GetNodeID(); SchedulerNodeReport report = sched.GetNodeReport(id); this.numContainers = 0; this.usedMemoryMB = 0; this.availMemoryMB = 0; if (report != null) { this.numContainers = report.GetNumContainers(); this.usedMemoryMB = report.GetUsedResource().GetMemory(); this.availMemoryMB = report.GetAvailableResource().GetMemory(); this.usedVirtualCores = report.GetUsedResource().GetVirtualCores(); this.availableVirtualCores = report.GetAvailableResource().GetVirtualCores(); } this.id = id.ToString(); this.rack = ni.GetRackName(); this.nodeHostName = ni.GetHostName(); this.state = ni.GetState(); this.nodeHTTPAddress = ni.GetHttpAddress(); this.lastHealthUpdate = ni.GetLastHealthReportTime(); this.healthReport = ni.GetHealthReport().ToString(); this.version = ni.GetNodeManagerVersion(); // add labels ICollection <string> labelSet = ni.GetNodeLabels(); if (labelSet != null) { Sharpen.Collections.AddAll(nodeLabels, labelSet); nodeLabels.Sort(); } }
/// <summary>Implements getLiveNodeManagers()</summary> /// <returns>JSON formatted string containing statuses of all node managers</returns> public virtual string GetLiveNodeManagers() { // RMNMInfoBeans ICollection <RMNode> nodes = this.rmContext.GetRMNodes().Values; IList <RMNMInfo.InfoMap> nodesInfo = new AList <RMNMInfo.InfoMap>(); foreach (RMNode ni in nodes) { SchedulerNodeReport report = scheduler.GetNodeReport(ni.GetNodeID()); RMNMInfo.InfoMap info = new RMNMInfo.InfoMap(); info["HostName"] = ni.GetHostName(); info["Rack"] = ni.GetRackName(); info["State"] = ni.GetState().ToString(); info["NodeId"] = ni.GetNodeID(); info["NodeHTTPAddress"] = ni.GetHttpAddress(); info["LastHealthUpdate"] = ni.GetLastHealthReportTime(); info["HealthReport"] = ni.GetHealthReport(); info["NodeManagerVersion"] = ni.GetNodeManagerVersion(); if (report != null) { info["NumContainers"] = report.GetNumContainers(); info["UsedMemoryMB"] = report.GetUsedResource().GetMemory(); info["AvailableMemoryMB"] = report.GetAvailableResource().GetMemory(); } nodesInfo.AddItem(info); } return(JSON.ToString(nodesInfo)); }
public FifoSchedulerInfo(ResourceManager rm) { // JAXB needs this RMContext rmContext = rm.GetRMContext(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); qName = fs.GetQueueInfo(string.Empty, false, false).GetQueueName(); QueueInfo qInfo = fs.GetQueueInfo(qName, true, true); this.usedCapacity = qInfo.GetCurrentCapacity(); this.capacity = qInfo.GetCapacity(); this.minQueueMemoryCapacity = fs.GetMinimumResourceCapability().GetMemory(); this.maxQueueMemoryCapacity = fs.GetMaximumResourceCapability().GetMemory(); this.qstate = qInfo.GetQueueState(); this.numNodes = rmContext.GetRMNodes().Count; this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; foreach (RMNode ni in rmContext.GetRMNodes().Values) { SchedulerNodeReport report = fs.GetNodeReport(ni.GetNodeID()); this.usedNodeCapacity += report.GetUsedResource().GetMemory(); this.availNodeCapacity += report.GetAvailableResource().GetMemory(); this.totalNodeCapacity += ni.GetTotalCapability().GetMemory(); this.numContainers += fs.GetNodeReport(ni.GetNodeID()).GetNumContainers(); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeInfoGeneric(MockNM nm, string state, string rack, string id, string nodeHostName, string nodeHTTPAddress, long lastHealthUpdate, string healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores , long availVirtualCores, string version) { RMNode node = rm.GetRMContext().GetRMNodes()[nm.GetNodeId()]; ResourceScheduler sched = rm.GetResourceScheduler(); SchedulerNodeReport report = sched.GetNodeReport(nm.GetNodeId()); WebServicesTestUtils.CheckStringMatch("state", node.GetState().ToString(), state); WebServicesTestUtils.CheckStringMatch("rack", node.GetRackName(), rack); WebServicesTestUtils.CheckStringMatch("id", nm.GetNodeId().ToString(), id); WebServicesTestUtils.CheckStringMatch("nodeHostName", nm.GetNodeId().GetHost(), nodeHostName ); WebServicesTestUtils.CheckStringMatch("healthReport", node.GetHealthReport().ToString (), healthReport); string expectedHttpAddress = nm.GetNodeId().GetHost() + ":" + nm.GetHttpPort(); WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress ); WebServicesTestUtils.CheckStringMatch("version", node.GetNodeManagerVersion(), version ); long expectedHealthUpdate = node.GetLastHealthReportTime(); NUnit.Framework.Assert.AreEqual("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate); if (report != null) { NUnit.Framework.Assert.AreEqual("numContainers doesn't match: " + numContainers, report.GetNumContainers(), numContainers); NUnit.Framework.Assert.AreEqual("usedMemoryMB doesn't match: " + usedMemoryMB, report .GetUsedResource().GetMemory(), usedMemoryMB); NUnit.Framework.Assert.AreEqual("availMemoryMB doesn't match: " + availMemoryMB, report.GetAvailableResource().GetMemory(), availMemoryMB); NUnit.Framework.Assert.AreEqual("usedVirtualCores doesn't match: " + usedVirtualCores , report.GetUsedResource().GetVirtualCores(), usedVirtualCores); NUnit.Framework.Assert.AreEqual("availVirtualCores doesn't match: " + availVirtualCores , report.GetAvailableResource().GetVirtualCores(), availVirtualCores); } }
public virtual void TestResourceOverCommit() { MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 4 * Gb); RMApp app1 = rm.SubmitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); // check node report, 2 GB used and 2 GB available NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // add request for containers am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 2 * Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } IList <Container> allocated1 = alloc1Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated1.Count); NUnit.Framework.Assert.AreEqual(2 * Gb, allocated1[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId()); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); // check node report, 4 GB used and 0 GB available NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); // check container is assigned with 2 GB. Container c1 = allocated1[0]; NUnit.Framework.Assert.AreEqual(2 * Gb, c1.GetResource().GetMemory()); // update node resource to 2 GB, so resource is over-consumed. IDictionary <NodeId, ResourceOption> nodeResourceMap = new Dictionary <NodeId, ResourceOption >(); nodeResourceMap[nm1.GetNodeId()] = ResourceOption.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2 * Gb, 1), -1); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.NewInstance(nodeResourceMap ); AdminService @as = rm.adminService; @as.UpdateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(-2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState .Complete, string.Empty, 0); nm1.ContainerStatus(containerStatus); int waitCount = 0; while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20) { Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count); NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses( ).Count); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); // As container return 2 GB back, the available resource becomes 0 again. NUnit.Framework.Assert.AreEqual(0 * Gb, report_nm1.GetAvailableResource().GetMemory ()); rm.Stop(); }
public virtual void Test() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); MockNM nm2 = rm.RegisterNode("127.0.0.2:5678", 4 * Gb); RMApp app1 = rm.SubmitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); RMApp app2 = rm.SubmitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); am2.RegisterAppAttempt(); SchedulerNodeReport report_nm2 = rm.GetResourceScheduler().GetNodeReport(nm2.GetNodeId ()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetUsedResource().GetMemory()); // add request for containers am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // add request for containers am2.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 3 * Gb, 0, 1); AllocateResponse alloc2Response = am2.Schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } while (alloc2Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 2..."); Sharpen.Thread.Sleep(1000); alloc2Response = am2.Schedule(); } // kick the scheduler, nothing given remaining 2 GB. nm2.NodeHeartbeat(true); IList <Container> allocated1 = alloc1Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated1.Count); NUnit.Framework.Assert.AreEqual(1 * Gb, allocated1[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId()); IList <Container> allocated2 = alloc2Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated2.Count); NUnit.Framework.Assert.AreEqual(3 * Gb, allocated2[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated2[0].GetNodeId()); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); report_nm2 = rm.GetResourceScheduler().GetNodeReport(nm2.GetNodeId()); NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetAvailableResource().GetMemory ()); NUnit.Framework.Assert.AreEqual(6 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetUsedResource().GetMemory()); Container c1 = allocated1[0]; NUnit.Framework.Assert.AreEqual(Gb, c1.GetResource().GetMemory()); ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState .Complete, string.Empty, 0); nm1.ContainerStatus(containerStatus); int waitCount = 0; while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20) { Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count); NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses( ).Count); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(5 * Gb, report_nm1.GetUsedResource().GetMemory()); rm.Stop(); }