public virtual void TestInvalidValidWhenOldRemoved()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest request1 = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(request1);
client.RemoveContainerRequest(request1);
AMRMClient.ContainerRequest request2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host3" }, null, Priority.NewInstance(1), true);
client.AddContainerRequest(request2);
client.RemoveContainerRequest(request2);
AMRMClient.ContainerRequest request3 = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(request3);
client.RemoveContainerRequest(request3);
AMRMClient.ContainerRequest request4 = new AMRMClient.ContainerRequest(capability
, null, new string[] { "rack1" }, Priority.NewInstance(1), true);
client.AddContainerRequest(request4);
}
private void VerifyResourceRequest(AMRMClientImpl <AMRMClient.ContainerRequest> client
, AMRMClient.ContainerRequest request, string location, bool expectedRelaxLocality
)
{
ResourceRequest ask = client.remoteRequestsTable[request.GetPriority()][location]
[request.GetCapability()].remoteRequest;
NUnit.Framework.Assert.AreEqual(location, ask.GetResourceName());
NUnit.Framework.Assert.AreEqual(1, ask.GetNumContainers());
NUnit.Framework.Assert.AreEqual(expectedRelaxLocality, ask.GetRelaxLocality());
}
public virtual void TestDisableLocalityRelaxation()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest nodeLevelRequest = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(nodeLevelRequest);
VerifyResourceRequest(client, nodeLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, nodeLevelRequest, "/rack1", false);
VerifyResourceRequest(client, nodeLevelRequest, "host1", true);
VerifyResourceRequest(client, nodeLevelRequest, "host2", true);
// Make sure we don't get any errors with two node-level requests at the
// same priority
AMRMClient.ContainerRequest nodeLevelRequest2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host2", "host3" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(nodeLevelRequest2);
AMRMClient.ContainerRequest rackLevelRequest = new AMRMClient.ContainerRequest(capability
, null, new string[] { "/rack3", "/rack4" }, Priority.NewInstance(2), false);
client.AddContainerRequest(rackLevelRequest);
VerifyResourceRequest(client, rackLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, rackLevelRequest, "/rack3", true);
VerifyResourceRequest(client, rackLevelRequest, "/rack4", true);
// Make sure we don't get any errors with two rack-level requests at the
// same priority
AMRMClient.ContainerRequest rackLevelRequest2 = new AMRMClient.ContainerRequest(capability
, null, new string[] { "/rack4", "/rack5" }, Priority.NewInstance(2), false);
client.AddContainerRequest(rackLevelRequest2);
AMRMClient.ContainerRequest bothLevelRequest = new AMRMClient.ContainerRequest(capability
, new string[] { "host3", "host4" }, new string[] { "rack1", "/otherrack" }, Priority
.NewInstance(3), false);
client.AddContainerRequest(bothLevelRequest);
VerifyResourceRequest(client, bothLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, bothLevelRequest, "rack1", true);
VerifyResourceRequest(client, bothLevelRequest, "/otherrack", true);
VerifyResourceRequest(client, bothLevelRequest, "host3", true);
VerifyResourceRequest(client, bothLevelRequest, "host4", true);
// Make sure we don't get any errors with two both-level requests at the
// same priority
AMRMClient.ContainerRequest bothLevelRequest2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host4", "host5" }, new string[] { "rack1", "/otherrack2" }, Priority
.NewInstance(3), false);
client.AddContainerRequest(bothLevelRequest2);
}
public virtual void TestFillInRacks()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability,
new string[] { "host1", "host2" }, new string[] { "/rack2" }, Priority.NewInstance
(1));
client.AddContainerRequest(request);
VerifyResourceRequest(client, request, "host1", true);
VerifyResourceRequest(client, request, "host2", true);
VerifyResourceRequest(client, request, "/rack1", true);
VerifyResourceRequest(client, request, "/rack2", true);
VerifyResourceRequest(client, request, ResourceRequest.Any, true);
}
// Test does major 6 steps verification.
// Step-1 : AMRMClient send allocate request for 2 container requests
// Step-2 : 2 containers are allocated by RM.
// Step-3 : AM Send 1 containerRequest(cRequest3) and 1 releaseRequests to
// RM
// Step-4 : On RM restart, AM(does not know RM is restarted) sends additional
// containerRequest(cRequest4) and blacklisted nodes.
// Intern RM send resync command
// Step-5 : Allocater after resync command & new containerRequest(cRequest5)
// Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientResendsRequestsOnRMRestart()
{
UserGroupInformation.SetLoginUser(null);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// Phase-1 Start 1st RM
TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm1.Start();
DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher();
// Submit the application
RMApp app = rm1.SubmitApp(1024);
dispatcher.Await();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
nm1.NodeHeartbeat(true);
// Node heartbeat
dispatcher.Await();
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
rm1.SendAMLaunched(appAttemptId);
dispatcher.Await();
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext
().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken
();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
// Step-1 : AMRMClient send allocate request for 2 ContainerRequest
// cRequest1 = h1 and cRequest2 = h1,h2
// blacklisted nodes = h2
AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl
(rm1);
amClient.Init(conf);
amClient.Start();
amClient.RegisterApplicationMaster("Host", 10000, string.Empty);
AMRMClient.ContainerRequest cRequest1 = CreateReq(1, 1024, new string[] { "h1" });
amClient.AddContainerRequest(cRequest1);
AMRMClient.ContainerRequest cRequest2 = CreateReq(1, 1024, new string[] { "h1", "h2" });
amClient.AddContainerRequest(cRequest2);
IList <string> blacklistAdditions = new AList <string>();
IList <string> blacklistRemoval = new AList <string>();
blacklistAdditions.AddItem("h2");
blacklistRemoval.AddItem("h10");
amClient.UpdateBlacklist(blacklistAdditions, blacklistRemoval);
blacklistAdditions.Remove("h2");
// remove from local list
AllocateResponse allocateResponse = amClient.Allocate(0.1f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
// Why 4 ask, why not 3 ask even h2 is blacklisted?
// On blacklisting host,applicationmaster has to remove ask request from
// remoterequest table.Here,test does not remove explicitely
AssertAsksAndReleases(4, 0, rm1);
AssertBlacklistAdditionsAndRemovals(1, 1, rm1);
// Step-2 : NM heart beat is sent.
// On 2nd AM allocate request, RM allocates 2 containers to AM
nm1.NodeHeartbeat(true);
// Node heartbeat
dispatcher.Await();
allocateResponse = amClient.Allocate(0.2f);
dispatcher.Await();
// 2 containers are allocated i.e for cRequest1 and cRequest2.
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 2, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(0, 0, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
IList <Container> allocatedContainers = allocateResponse.GetAllocatedContainers();
// removed allocated container requests
amClient.RemoveContainerRequest(cRequest1);
amClient.RemoveContainerRequest(cRequest2);
allocateResponse = amClient.Allocate(0.2f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(4, 0, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
// Step-3 : Send 1 containerRequest and 1 releaseRequests to RM
AMRMClient.ContainerRequest cRequest3 = CreateReq(1, 1024, new string[] { "h1" });
amClient.AddContainerRequest(cRequest3);
int pendingRelease = 0;
IEnumerator <Container> it = allocatedContainers.GetEnumerator();
while (it.HasNext())
{
amClient.ReleaseAssignedContainer(it.Next().GetId());
pendingRelease++;
it.Remove();
break;
}
// remove one container
allocateResponse = amClient.Allocate(0.3f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(3, pendingRelease, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
int completedContainer = allocateResponse.GetCompletedContainersStatuses().Count;
pendingRelease -= completedContainer;
// Phase-2 start 2nd RM is up
TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm2.Start();
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2);
dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction());
// new NM to represent NM re-register
nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService());
nm1.RegisterNode();
nm1.NodeHeartbeat(true);
dispatcher.Await();
blacklistAdditions.AddItem("h3");
amClient.UpdateBlacklist(blacklistAdditions, null);
blacklistAdditions.Remove("h3");
it = allocatedContainers.GetEnumerator();
while (it.HasNext())
{
amClient.ReleaseAssignedContainer(it.Next().GetId());
pendingRelease++;
it.Remove();
}
AMRMClient.ContainerRequest cRequest4 = CreateReq(1, 1024, new string[] { "h1", "h2" });
amClient.AddContainerRequest(cRequest4);
// Step-4 : On RM restart, AM(does not know RM is restarted) sends
// additional
// containerRequest and blacklisted nodes.
// Intern RM send resync command,AMRMClient resend allocate request
allocateResponse = amClient.Allocate(0.3f);
dispatcher.Await();
completedContainer = allocateResponse.GetCompletedContainersStatuses().Count;
pendingRelease -= completedContainer;
AssertAsksAndReleases(4, pendingRelease, rm2);
AssertBlacklistAdditionsAndRemovals(2, 0, rm2);
AMRMClient.ContainerRequest cRequest5 = CreateReq(1, 1024, new string[] { "h1", "h2"
, "h3" });
amClient.AddContainerRequest(cRequest5);
// Step-5 : Allocater after resync command
allocateResponse = amClient.Allocate(0.5f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(5, 0, rm2);
AssertBlacklistAdditionsAndRemovals(0, 0, rm2);
int noAssignedContainer = 0;
int count = 5;
while (count-- > 0)
{
nm1.NodeHeartbeat(true);
dispatcher.Await();
allocateResponse = amClient.Allocate(0.5f);
dispatcher.Await();
noAssignedContainer += allocateResponse.GetAllocatedContainers().Count;
if (noAssignedContainer == 3)
{
break;
}
Sharpen.Thread.Sleep(1000);
}
// Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5
NUnit.Framework.Assert.AreEqual("Number of container should be 3", 3, noAssignedContainer
);
amClient.Stop();
rm1.Stop();
rm2.Stop();
}
