public virtual void TestMaxAllocationAfterUpdateNodeResource()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(4096, 10);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(512, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 2);
RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0));
VerifyMaximumResourceCapability(resource2, scheduler);
// decrease node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// decrease node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
}
finally
{
rm.Stop();
}
}
/// <exception cref="System.Exception"/>
private void TestMaximumAllocationMemoryHelper(AbstractYarnScheduler scheduler, int
node1MaxMemory, int node2MaxMemory, int node3MaxMemory, params int[] expectedMaxMemory
)
{
NUnit.Framework.Assert.AreEqual(6, expectedMaxMemory.Length);
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
int maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[0], maxMemory);
RMNode node1 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node1MaxMemory),
1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[1], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[2], maxMemory);
RMNode node2 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node2MaxMemory),
2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[3], maxMemory);
RMNode node3 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node3MaxMemory),
3, "127.0.0.4");
scheduler.Handle(new NodeAddedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(2, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[4], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[5], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
}