public virtual void TestHeadroom()
{
FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>();
Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock(
));
FSLeafQueue mockQueue = Org.Mockito.Mockito.Mock <FSLeafQueue>();
Resource queueMaxResources = Resource.NewInstance(5 * 1024, 3);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource
(4096, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract(
queueFairShare, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources
.Subtract(queueMaxResources, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(8192, 8);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource
(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract
(clusterResource, clusterUsage);
QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources);
Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare);
Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage);
Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource
);
Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn
(clusterUsage);
Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId,
"user1", mockQueue, null, rmContext);
// Min of Memory and CPU across cluster and queue is used in
// DominantResourceFairnessPolicy
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(DominantResourceFairnessPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores
(), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores
()));
// Fair and Fifo ignore CPU of queue, so use cluster available CPU
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FairSharePolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FifoPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
}
/// <exception cref="System.Exception"/>
public virtual void TestIsStarvedForFairShare()
{
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<weight>.2</weight>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<weight>.8</weight>");
@out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>");
@out.WriteLine("<queue name=\"queueB1\">");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB2\">");
@out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>");
@out.WriteLine("</queue>");
@out.WriteLine("</queue>");
@out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>"
);
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10),
1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
scheduler.Update();
// Queue A wants 4 * 1024. Node update gives this all to A
CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4);
scheduler.Update();
NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
for (int i = 0; i < 4; i++)
{
scheduler.Handle(nodeEvent2);
}
QueueManager queueMgr = scheduler.GetQueueManager();
FSLeafQueue queueA = queueMgr.GetLeafQueue("queueA", false);
NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 want 3 * 1024
CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3);
CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3);
scheduler.Update();
for (int i_1 = 0; i_1 < 4; i_1++)
{
scheduler.Handle(nodeEvent2);
}
FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false);
FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false);
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory());
// For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share
// threshold is 1.6 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
// For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share
// threshold is 2.4 * 1024
NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare());
// Node checks in again
scheduler.Handle(nodeEvent2);
scheduler.Handle(nodeEvent2);
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 usages go to 3 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare());
}