public virtual void TestUpdateMaxAllocationUsesTotal()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 10);
SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
, 8080));
Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
, 8081));
Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
scheduler.UpdateMaximumAllocation(mockNode1, true);
VerifyMaximumResourceCapability(fullResource1, scheduler);
scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
scheduler.UpdateMaximumAllocation(mockNode2, true);
VerifyMaximumResourceCapability(fullResource2, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode2, false);
VerifyMaximumResourceCapability(fullResource1, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode1, false);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
}
finally
{
rm.Stop();
}
}
private void VerifyMaximumResourceCapability(Org.Apache.Hadoop.Yarn.Api.Records.Resource
expectedMaximumResource, AbstractYarnScheduler scheduler)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource schedulerMaximumResourceCapability =
scheduler.GetMaximumResourceCapability();
NUnit.Framework.Assert.AreEqual(expectedMaximumResource.GetMemory(), schedulerMaximumResourceCapability
.GetMemory());
NUnit.Framework.Assert.AreEqual(expectedMaximumResource.GetVirtualCores(), schedulerMaximumResourceCapability
.GetVirtualCores());
}
public virtual void TestMaxAllocationAfterUpdateNodeResource()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(4096, 10);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(512, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 2);
RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0));
VerifyMaximumResourceCapability(resource2, scheduler);
// decrease node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// decrease node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
}
finally
{
rm.Stop();
}
}
/// <exception cref="System.Exception"/>
private void TestMaximumAllocationMemoryHelper(AbstractYarnScheduler scheduler, int
node1MaxMemory, int node2MaxMemory, int node3MaxMemory, params int[] expectedMaxMemory
)
{
NUnit.Framework.Assert.AreEqual(6, expectedMaxMemory.Length);
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
int maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[0], maxMemory);
RMNode node1 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node1MaxMemory),
1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[1], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[2], maxMemory);
RMNode node2 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node2MaxMemory),
2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[3], maxMemory);
RMNode node3 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node3MaxMemory),
3, "127.0.0.4");
scheduler.Handle(new NodeAddedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(2, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[4], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[5], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
}
public _TimerTask_450(AbstractYarnScheduler <T, N> _enclosing)
{
this._enclosing = _enclosing;
}
