public virtual void TestInvalidValidWhenOldRemoved() { AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest >(); Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping)); client.Init(conf); Resource capability = Resource.NewInstance(1024, 1); AMRMClient.ContainerRequest request1 = new AMRMClient.ContainerRequest(capability , new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false); client.AddContainerRequest(request1); client.RemoveContainerRequest(request1); AMRMClient.ContainerRequest request2 = new AMRMClient.ContainerRequest(capability , new string[] { "host3" }, null, Priority.NewInstance(1), true); client.AddContainerRequest(request2); client.RemoveContainerRequest(request2); AMRMClient.ContainerRequest request3 = new AMRMClient.ContainerRequest(capability , new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false); client.AddContainerRequest(request3); client.RemoveContainerRequest(request3); AMRMClient.ContainerRequest request4 = new AMRMClient.ContainerRequest(capability , null, new string[] { "rack1" }, Priority.NewInstance(1), true); client.AddContainerRequest(request4); }
private void VerifyResourceRequest(AMRMClientImpl <AMRMClient.ContainerRequest> client , AMRMClient.ContainerRequest request, string location, bool expectedRelaxLocality ) { ResourceRequest ask = client.remoteRequestsTable[request.GetPriority()][location] [request.GetCapability()].remoteRequest; NUnit.Framework.Assert.AreEqual(location, ask.GetResourceName()); NUnit.Framework.Assert.AreEqual(1, ask.GetNumContainers()); NUnit.Framework.Assert.AreEqual(expectedRelaxLocality, ask.GetRelaxLocality()); }
public virtual void TestDisableLocalityRelaxation() { AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest >(); Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping)); client.Init(conf); Resource capability = Resource.NewInstance(1024, 1); AMRMClient.ContainerRequest nodeLevelRequest = new AMRMClient.ContainerRequest(capability , new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false); client.AddContainerRequest(nodeLevelRequest); VerifyResourceRequest(client, nodeLevelRequest, ResourceRequest.Any, false); VerifyResourceRequest(client, nodeLevelRequest, "/rack1", false); VerifyResourceRequest(client, nodeLevelRequest, "host1", true); VerifyResourceRequest(client, nodeLevelRequest, "host2", true); // Make sure we don't get any errors with two node-level requests at the // same priority AMRMClient.ContainerRequest nodeLevelRequest2 = new AMRMClient.ContainerRequest(capability , new string[] { "host2", "host3" }, null, Priority.NewInstance(1), false); client.AddContainerRequest(nodeLevelRequest2); AMRMClient.ContainerRequest rackLevelRequest = new AMRMClient.ContainerRequest(capability , null, new string[] { "/rack3", "/rack4" }, Priority.NewInstance(2), false); client.AddContainerRequest(rackLevelRequest); VerifyResourceRequest(client, rackLevelRequest, ResourceRequest.Any, false); VerifyResourceRequest(client, rackLevelRequest, "/rack3", true); VerifyResourceRequest(client, rackLevelRequest, "/rack4", true); // Make sure we don't get any errors with two rack-level requests at the // same priority AMRMClient.ContainerRequest rackLevelRequest2 = new AMRMClient.ContainerRequest(capability , null, new string[] { "/rack4", "/rack5" }, Priority.NewInstance(2), false); client.AddContainerRequest(rackLevelRequest2); AMRMClient.ContainerRequest bothLevelRequest = new AMRMClient.ContainerRequest(capability , new string[] { "host3", "host4" }, new string[] { "rack1", "/otherrack" }, Priority .NewInstance(3), false); client.AddContainerRequest(bothLevelRequest); VerifyResourceRequest(client, bothLevelRequest, ResourceRequest.Any, false); VerifyResourceRequest(client, bothLevelRequest, "rack1", true); VerifyResourceRequest(client, bothLevelRequest, "/otherrack", true); VerifyResourceRequest(client, bothLevelRequest, "host3", true); VerifyResourceRequest(client, bothLevelRequest, "host4", true); // Make sure we don't get any errors with two both-level requests at the // same priority AMRMClient.ContainerRequest bothLevelRequest2 = new AMRMClient.ContainerRequest(capability , new string[] { "host4", "host5" }, new string[] { "rack1", "/otherrack2" }, Priority .NewInstance(3), false); client.AddContainerRequest(bothLevelRequest2); }
public virtual void TestFillInRacks() { AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest >(); Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping)); client.Init(conf); Resource capability = Resource.NewInstance(1024, 1); AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, new string[] { "host1", "host2" }, new string[] { "/rack2" }, Priority.NewInstance (1)); client.AddContainerRequest(request); VerifyResourceRequest(client, request, "host1", true); VerifyResourceRequest(client, request, "host2", true); VerifyResourceRequest(client, request, "/rack1", true); VerifyResourceRequest(client, request, "/rack2", true); VerifyResourceRequest(client, request, ResourceRequest.Any, true); }
// Test does major 6 steps verification. // Step-1 : AMRMClient send allocate request for 2 container requests // Step-2 : 2 containers are allocated by RM. // Step-3 : AM Send 1 containerRequest(cRequest3) and 1 releaseRequests to // RM // Step-4 : On RM restart, AM(does not know RM is restarted) sends additional // containerRequest(cRequest4) and blacklisted nodes. // Intern RM send resync command // Step-5 : Allocater after resync command & new containerRequest(cRequest5) // Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5 /// <exception cref="System.Exception"/> public virtual void TestAMRMClientResendsRequestsOnRMRestart() { UserGroupInformation.SetLoginUser(null); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // Phase-1 Start 1st RM TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm1.Start(); DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher(); // Submit the application RMApp app = rm1.SubmitApp(1024); dispatcher.Await(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId(); rm1.SendAMLaunched(appAttemptId); dispatcher.Await(); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext ().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken (); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ugi.AddTokenIdentifier(token.DecodeIdentifier()); // Step-1 : AMRMClient send allocate request for 2 ContainerRequest // cRequest1 = h1 and cRequest2 = h1,h2 // blacklisted nodes = h2 AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl (rm1); amClient.Init(conf); amClient.Start(); amClient.RegisterApplicationMaster("Host", 10000, string.Empty); AMRMClient.ContainerRequest cRequest1 = CreateReq(1, 1024, new string[] { "h1" }); amClient.AddContainerRequest(cRequest1); AMRMClient.ContainerRequest cRequest2 = CreateReq(1, 1024, new string[] { "h1", "h2" }); amClient.AddContainerRequest(cRequest2); IList <string> blacklistAdditions = new AList <string>(); IList <string> blacklistRemoval = new AList <string>(); blacklistAdditions.AddItem("h2"); blacklistRemoval.AddItem("h10"); amClient.UpdateBlacklist(blacklistAdditions, blacklistRemoval); blacklistAdditions.Remove("h2"); // remove from local list AllocateResponse allocateResponse = amClient.Allocate(0.1f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); // Why 4 ask, why not 3 ask even h2 is blacklisted? // On blacklisting host,applicationmaster has to remove ask request from // remoterequest table.Here,test does not remove explicitely AssertAsksAndReleases(4, 0, rm1); AssertBlacklistAdditionsAndRemovals(1, 1, rm1); // Step-2 : NM heart beat is sent. // On 2nd AM allocate request, RM allocates 2 containers to AM nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); allocateResponse = amClient.Allocate(0.2f); dispatcher.Await(); // 2 containers are allocated i.e for cRequest1 and cRequest2. NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 2, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(0, 0, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); IList <Container> allocatedContainers = allocateResponse.GetAllocatedContainers(); // removed allocated container requests amClient.RemoveContainerRequest(cRequest1); amClient.RemoveContainerRequest(cRequest2); allocateResponse = amClient.Allocate(0.2f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(4, 0, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); // Step-3 : Send 1 containerRequest and 1 releaseRequests to RM AMRMClient.ContainerRequest cRequest3 = CreateReq(1, 1024, new string[] { "h1" }); amClient.AddContainerRequest(cRequest3); int pendingRelease = 0; IEnumerator <Container> it = allocatedContainers.GetEnumerator(); while (it.HasNext()) { amClient.ReleaseAssignedContainer(it.Next().GetId()); pendingRelease++; it.Remove(); break; } // remove one container allocateResponse = amClient.Allocate(0.3f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(3, pendingRelease, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); int completedContainer = allocateResponse.GetCompletedContainersStatuses().Count; pendingRelease -= completedContainer; // Phase-2 start 2nd RM is up TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm2.Start(); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); ((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2); dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction()); // new NM to represent NM re-register nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); dispatcher.Await(); blacklistAdditions.AddItem("h3"); amClient.UpdateBlacklist(blacklistAdditions, null); blacklistAdditions.Remove("h3"); it = allocatedContainers.GetEnumerator(); while (it.HasNext()) { amClient.ReleaseAssignedContainer(it.Next().GetId()); pendingRelease++; it.Remove(); } AMRMClient.ContainerRequest cRequest4 = CreateReq(1, 1024, new string[] { "h1", "h2" }); amClient.AddContainerRequest(cRequest4); // Step-4 : On RM restart, AM(does not know RM is restarted) sends // additional // containerRequest and blacklisted nodes. // Intern RM send resync command,AMRMClient resend allocate request allocateResponse = amClient.Allocate(0.3f); dispatcher.Await(); completedContainer = allocateResponse.GetCompletedContainersStatuses().Count; pendingRelease -= completedContainer; AssertAsksAndReleases(4, pendingRelease, rm2); AssertBlacklistAdditionsAndRemovals(2, 0, rm2); AMRMClient.ContainerRequest cRequest5 = CreateReq(1, 1024, new string[] { "h1", "h2" , "h3" }); amClient.AddContainerRequest(cRequest5); // Step-5 : Allocater after resync command allocateResponse = amClient.Allocate(0.5f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(5, 0, rm2); AssertBlacklistAdditionsAndRemovals(0, 0, rm2); int noAssignedContainer = 0; int count = 5; while (count-- > 0) { nm1.NodeHeartbeat(true); dispatcher.Await(); allocateResponse = amClient.Allocate(0.5f); dispatcher.Await(); noAssignedContainer += allocateResponse.GetAllocatedContainers().Count; if (noAssignedContainer == 3) { break; } Sharpen.Thread.Sleep(1000); } // Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5 NUnit.Framework.Assert.AreEqual("Number of container should be 3", 3, noAssignedContainer ); amClient.Stop(); rm1.Stop(); rm2.Stop(); }