/// <exception cref="System.Exception"/> public virtual void TestInvalidContainerReleaseRequest() { MockRM rm = new MockRM(conf); try { rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); // Submit an application RMApp app1 = rm.SubmitApp(1024); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0); RMApp app2 = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); am2.RegisterAppAttempt(); // Now trying to release container allocated for app1 -> appAttempt1. ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId(); am2.AddContainerToBeReleased(cId); try { am2.Schedule(); NUnit.Framework.Assert.Fail("Exception was expected!!"); } catch (InvalidContainerReleaseException e) { StringBuilder sb = new StringBuilder("Cannot release container : "); sb.Append(cId.ToString()); sb.Append(" not belonging to this application attempt : "); sb.Append(attempt2.GetAppAttemptId().ToString()); NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString())); } } finally { if (rm != null) { rm.Stop(); } } }
/// <exception cref="System.Exception"/> public virtual void TestRMIdentifierOnContainerAllocation() { MockRM rm = new MockRM(conf); rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); // Submit an application RMApp app1 = rm.SubmitApp(2048); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } // assert RMIdentifer is set properly in allocated containers Container allocatedContainer = alloc1Response.GetAllocatedContainers()[0]; ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer .GetContainerToken()); NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier ()); rm.Stop(); }
public virtual void TestResourceOverCommit() { MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 4 * Gb); RMApp app1 = rm.SubmitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); // check node report, 2 GB used and 2 GB available NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // add request for containers am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 2 * Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } IList <Container> allocated1 = alloc1Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated1.Count); NUnit.Framework.Assert.AreEqual(2 * Gb, allocated1[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId()); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); // check node report, 4 GB used and 0 GB available NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); // check container is assigned with 2 GB. Container c1 = allocated1[0]; NUnit.Framework.Assert.AreEqual(2 * Gb, c1.GetResource().GetMemory()); // update node resource to 2 GB, so resource is over-consumed. IDictionary <NodeId, ResourceOption> nodeResourceMap = new Dictionary <NodeId, ResourceOption >(); nodeResourceMap[nm1.GetNodeId()] = ResourceOption.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2 * Gb, 1), -1); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.NewInstance(nodeResourceMap ); AdminService @as = rm.adminService; @as.UpdateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(-2 * Gb, report_nm1.GetAvailableResource().GetMemory ()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState .Complete, string.Empty, 0); nm1.ContainerStatus(containerStatus); int waitCount = 0; while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20) { Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count); NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses( ).Count); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); // As container return 2 GB back, the available resource becomes 0 again. NUnit.Framework.Assert.AreEqual(0 * Gb, report_nm1.GetAvailableResource().GetMemory ()); rm.Stop(); }
public virtual void Test() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); MockNM nm2 = rm.RegisterNode("127.0.0.2:5678", 4 * Gb); RMApp app1 = rm.SubmitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory()); RMApp app2 = rm.SubmitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); am2.RegisterAppAttempt(); SchedulerNodeReport report_nm2 = rm.GetResourceScheduler().GetNodeReport(nm2.GetNodeId ()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetUsedResource().GetMemory()); // add request for containers am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // add request for containers am2.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 3 * Gb, 0, 1); AllocateResponse alloc2Response = am2.Schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } while (alloc2Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 2..."); Sharpen.Thread.Sleep(1000); alloc2Response = am2.Schedule(); } // kick the scheduler, nothing given remaining 2 GB. nm2.NodeHeartbeat(true); IList <Container> allocated1 = alloc1Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated1.Count); NUnit.Framework.Assert.AreEqual(1 * Gb, allocated1[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId()); IList <Container> allocated2 = alloc2Response.GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(1, allocated2.Count); NUnit.Framework.Assert.AreEqual(3 * Gb, allocated2[0].GetResource().GetMemory()); NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated2[0].GetNodeId()); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); report_nm2 = rm.GetResourceScheduler().GetNodeReport(nm2.GetNodeId()); NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetAvailableResource().GetMemory ()); NUnit.Framework.Assert.AreEqual(6 * Gb, report_nm1.GetUsedResource().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm2.GetUsedResource().GetMemory()); Container c1 = allocated1[0]; NUnit.Framework.Assert.AreEqual(Gb, c1.GetResource().GetMemory()); ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState .Complete, string.Empty, 0); nm1.ContainerStatus(containerStatus); int waitCount = 0; while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20) { Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count); NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses( ).Count); report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId()); NUnit.Framework.Assert.AreEqual(5 * Gb, report_nm1.GetUsedResource().GetMemory()); rm.Stop(); }