public virtual void TestInvalidValidWhenOldRemoved()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest request1 = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(request1);
client.RemoveContainerRequest(request1);
AMRMClient.ContainerRequest request2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host3" }, null, Priority.NewInstance(1), true);
client.AddContainerRequest(request2);
client.RemoveContainerRequest(request2);
AMRMClient.ContainerRequest request3 = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(request3);
client.RemoveContainerRequest(request3);
AMRMClient.ContainerRequest request4 = new AMRMClient.ContainerRequest(capability
, null, new string[] { "rack1" }, Priority.NewInstance(1), true);
client.AddContainerRequest(request4);
}
private void VerifyResourceRequest(AMRMClientImpl <AMRMClient.ContainerRequest> client
, AMRMClient.ContainerRequest request, string location, bool expectedRelaxLocality
)
{
ResourceRequest ask = client.remoteRequestsTable[request.GetPriority()][location]
[request.GetCapability()].remoteRequest;
NUnit.Framework.Assert.AreEqual(location, ask.GetResourceName());
NUnit.Framework.Assert.AreEqual(1, ask.GetNumContainers());
NUnit.Framework.Assert.AreEqual(expectedRelaxLocality, ask.GetRelaxLocality());
}
internal ResourceRequestInfo(AMRMClientImpl <T> _enclosing, Priority priority, string
resourceName, Resource capability, bool relaxLocality)
{
this._enclosing = _enclosing;
// blacklistedNodes is required for keeping history of blacklisted nodes that
// are sent to RM. On RESYNC command from RM, blacklistedNodes are used to get
// current blacklisted nodes and send back to RM.
this.remoteRequest = ResourceRequest.NewInstance(priority, resourceName, capability
, 0);
this.remoteRequest.SetRelaxLocality(relaxLocality);
this.containerRequests = new LinkedHashSet <T>();
}
public virtual void TestDisableLocalityRelaxation()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest nodeLevelRequest = new AMRMClient.ContainerRequest(capability
, new string[] { "host1", "host2" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(nodeLevelRequest);
VerifyResourceRequest(client, nodeLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, nodeLevelRequest, "/rack1", false);
VerifyResourceRequest(client, nodeLevelRequest, "host1", true);
VerifyResourceRequest(client, nodeLevelRequest, "host2", true);
// Make sure we don't get any errors with two node-level requests at the
// same priority
AMRMClient.ContainerRequest nodeLevelRequest2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host2", "host3" }, null, Priority.NewInstance(1), false);
client.AddContainerRequest(nodeLevelRequest2);
AMRMClient.ContainerRequest rackLevelRequest = new AMRMClient.ContainerRequest(capability
, null, new string[] { "/rack3", "/rack4" }, Priority.NewInstance(2), false);
client.AddContainerRequest(rackLevelRequest);
VerifyResourceRequest(client, rackLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, rackLevelRequest, "/rack3", true);
VerifyResourceRequest(client, rackLevelRequest, "/rack4", true);
// Make sure we don't get any errors with two rack-level requests at the
// same priority
AMRMClient.ContainerRequest rackLevelRequest2 = new AMRMClient.ContainerRequest(capability
, null, new string[] { "/rack4", "/rack5" }, Priority.NewInstance(2), false);
client.AddContainerRequest(rackLevelRequest2);
AMRMClient.ContainerRequest bothLevelRequest = new AMRMClient.ContainerRequest(capability
, new string[] { "host3", "host4" }, new string[] { "rack1", "/otherrack" }, Priority
.NewInstance(3), false);
client.AddContainerRequest(bothLevelRequest);
VerifyResourceRequest(client, bothLevelRequest, ResourceRequest.Any, false);
VerifyResourceRequest(client, bothLevelRequest, "rack1", true);
VerifyResourceRequest(client, bothLevelRequest, "/otherrack", true);
VerifyResourceRequest(client, bothLevelRequest, "host3", true);
VerifyResourceRequest(client, bothLevelRequest, "host4", true);
// Make sure we don't get any errors with two both-level requests at the
// same priority
AMRMClient.ContainerRequest bothLevelRequest2 = new AMRMClient.ContainerRequest(capability
, new string[] { "host4", "host5" }, new string[] { "rack1", "/otherrack2" }, Priority
.NewInstance(3), false);
client.AddContainerRequest(bothLevelRequest2);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private ICollection <Container> AllocateContainers(AMRMClientImpl <AMRMClient.ContainerRequest
> rmClient, int num)
{
// setup container request
Resource capability = Resource.NewInstance(1024, 0);
Priority priority = Priority.NewInstance(0);
string node = nodeReports[0].GetNodeId().GetHost();
string rack = nodeReports[0].GetRackName();
string[] nodes = new string[] { node };
string[] racks = new string[] { rack };
for (int i = 0; i < num; ++i)
{
rmClient.AddContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks
, priority));
}
int containersRequestedAny = rmClient.remoteRequestsTable[priority][ResourceRequest
.Any][capability].remoteRequest.GetNumContainers();
// RM should allocate container within 2 calls to allocate()
int allocatedContainerCount = 0;
int iterationsLeft = 2;
ICollection <Container> containers = new TreeSet <Container>();
while (allocatedContainerCount < containersRequestedAny && iterationsLeft > 0)
{
AllocateResponse allocResponse = rmClient.Allocate(0.1f);
allocatedContainerCount += allocResponse.GetAllocatedContainers().Count;
foreach (Container container in allocResponse.GetAllocatedContainers())
{
containers.AddItem(container);
}
if (!allocResponse.GetNMTokens().IsEmpty())
{
foreach (NMToken token in allocResponse.GetNMTokens())
{
rmClient.GetNMTokenCache().SetToken(token.GetNodeId().ToString(), token.GetToken(
));
}
}
if (allocatedContainerCount < containersRequestedAny)
{
// sleep to let NM's heartbeat to RM and trigger allocations
Sleep(1000);
}
--iterationsLeft;
}
return(containers);
}
public virtual void TestFillInRacks()
{
AMRMClientImpl <AMRMClient.ContainerRequest> client = new AMRMClientImpl <AMRMClient.ContainerRequest
>();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestAMRMClientContainerRequest.MyResolver), typeof(DNSToSwitchMapping));
client.Init(conf);
Resource capability = Resource.NewInstance(1024, 1);
AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability,
new string[] { "host1", "host2" }, new string[] { "/rack2" }, Priority.NewInstance
(1));
client.AddContainerRequest(request);
VerifyResourceRequest(client, request, "host1", true);
VerifyResourceRequest(client, request, "host2", true);
VerifyResourceRequest(client, request, "/rack1", true);
VerifyResourceRequest(client, request, "/rack2", true);
VerifyResourceRequest(client, request, ResourceRequest.Any, true);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
internal ResourceReverseMemoryThenCpuComparator(AMRMClientImpl <T> _enclosing)
{
this._enclosing = _enclosing;
}