public virtual void TestHeadroom() { FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>(); Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock( )); FSLeafQueue mockQueue = Org.Mockito.Mockito.Mock <FSLeafQueue>(); Resource queueMaxResources = Resource.NewInstance(5 * 1024, 3); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource (4096, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract( queueFairShare, queueUsage); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources .Subtract(queueMaxResources, queueUsage); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (8192, 8); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource (2048, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract (clusterResource, clusterUsage); QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources); Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare); Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage); Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource ); Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn (clusterUsage); Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics ); ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1); RMContext rmContext = resourceManager.GetRMContext(); FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue, null, rmContext); // Min of Memory and CPU across cluster and queue is used in // DominantResourceFairnessPolicy Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(DominantResourceFairnessPolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores (), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores ())); // Fair and Fifo ignore CPU of queue, so use cluster available CPU Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(FairSharePolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores (), queueMaxResourcesAvailable.GetVirtualCores())); Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(FifoPolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores (), queueMaxResourcesAvailable.GetVirtualCores())); }
/// <exception cref="System.Exception"/> public virtual void TestIsStarvedForFairShare() { conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"queueA\">"); @out.WriteLine("<weight>.2</weight>"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB\">"); @out.WriteLine("<weight>.8</weight>"); @out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>"); @out.WriteLine("<queue name=\"queueB1\">"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB2\">"); @out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>"); @out.WriteLine("</queue>"); @out.WriteLine("</queue>"); @out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>" ); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); scheduler.Update(); // Queue A wants 4 * 1024. Node update gives this all to A CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4); scheduler.Update(); NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); for (int i = 0; i < 4; i++) { scheduler.Handle(nodeEvent2); } QueueManager queueMgr = scheduler.GetQueueManager(); FSLeafQueue queueA = queueMgr.GetLeafQueue("queueA", false); NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory()); // Both queue B1 and queue B2 want 3 * 1024 CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3); CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3); scheduler.Update(); for (int i_1 = 0; i_1 < 4; i_1++) { scheduler.Handle(nodeEvent2); } FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false); FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false); NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory()); // For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share // threshold is 1.6 * 1024 NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare()); // For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share // threshold is 2.4 * 1024 NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare()); // Node checks in again scheduler.Handle(nodeEvent2); scheduler.Handle(nodeEvent2); NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory()); NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory()); // Both queue B1 and queue B2 usages go to 3 * 1024 NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare()); NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare()); }