public virtual void TestHeadroom() { FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>(); Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock( )); FSLeafQueue mockQueue = Org.Mockito.Mockito.Mock <FSLeafQueue>(); Resource queueMaxResources = Resource.NewInstance(5 * 1024, 3); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource (4096, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract( queueFairShare, queueUsage); Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources .Subtract(queueMaxResources, queueUsage); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (8192, 8); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource (2048, 2); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract (clusterResource, clusterUsage); QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>(); Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources); Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare); Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage); Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource ); Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn (clusterUsage); Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics ); ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1); RMContext rmContext = resourceManager.GetRMContext(); FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue, null, rmContext); // Min of Memory and CPU across cluster and queue is used in // DominantResourceFairnessPolicy Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(DominantResourceFairnessPolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores (), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores ())); // Fair and Fifo ignore CPU of queue, so use cluster available CPU Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(FairSharePolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores (), queueMaxResourcesAvailable.GetVirtualCores())); Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance (typeof(FifoPolicy))); VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores (), queueMaxResourcesAvailable.GetVirtualCores())); }
public virtual QueueInfo GetQueueInfo(bool includeChildQueues, bool recursive) { QueueInfo queueInfo = recordFactory.NewRecordInstance <QueueInfo>(); queueInfo.SetQueueName(GetQueueName()); if (scheduler.GetClusterResource().GetMemory() == 0) { queueInfo.SetCapacity(0.0f); } else { queueInfo.SetCapacity((float)GetFairShare().GetMemory() / scheduler.GetClusterResource ().GetMemory()); } if (GetFairShare().GetMemory() == 0) { queueInfo.SetCurrentCapacity(0.0f); } else { queueInfo.SetCurrentCapacity((float)GetResourceUsage().GetMemory() / GetFairShare ().GetMemory()); } AList <QueueInfo> childQueueInfos = new AList <QueueInfo>(); if (includeChildQueues) { ICollection <Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueue> childQueues = GetChildQueues(); foreach (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueue child in childQueues) { childQueueInfos.AddItem(child.GetQueueInfo(recursive, recursive)); } } queueInfo.SetChildQueues(childQueueInfos); queueInfo.SetQueueState(QueueState.Running); return(queueInfo); }
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration(); queueName = queue.GetName(); schedulingPolicy = queue.GetPolicy().GetName(); clusterResources = new ResourceInfo(scheduler.GetClusterResource()); usedResources = new ResourceInfo(queue.GetResourceUsage()); fractionMemUsed = (float)usedResources.GetMemory() / clusterResources.GetMemory(); steadyFairResources = new ResourceInfo(queue.GetSteadyFairShare()); fairResources = new ResourceInfo(queue.GetFairShare()); minResources = new ResourceInfo(queue.GetMinShare()); maxResources = new ResourceInfo(queue.GetMaxShare()); maxResources = new ResourceInfo(Resources.ComponentwiseMin(queue.GetMaxShare(), scheduler .GetClusterResource())); fractionMemSteadyFairShare = (float)steadyFairResources.GetMemory() / clusterResources .GetMemory(); fractionMemFairShare = (float)fairResources.GetMemory() / clusterResources.GetMemory (); fractionMemMinShare = (float)minResources.GetMemory() / clusterResources.GetMemory (); fractionMemMaxShare = (float)maxResources.GetMemory() / clusterResources.GetMemory (); maxApps = allocConf.GetQueueMaxApps(queueName); childQueues = new AList <Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.Dao.FairSchedulerQueueInfo >(); if (allocConf.IsReservable(queueName) && !allocConf.GetShowReservationAsQueues(queueName )) { return; } ICollection <FSQueue> children = queue.GetChildQueues(); foreach (FSQueue child in children) { if (child is FSLeafQueue) { childQueues.AddItem(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler) ); } else { childQueues.AddItem(new Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.Dao.FairSchedulerQueueInfo (child, scheduler)); } } }
public virtual void UpdateAllocationConfiguration(AllocationConfiguration queueConf ) { // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist foreach (string name in queueConf.GetConfiguredQueues()[FSQueueType.Leaf]) { if (RemoveEmptyIncompatibleQueues(name, FSQueueType.Leaf)) { GetLeafQueue(name, true); } } // At this point all leaves and 'parents with at least one child' would have been created. // Now create parents with no configured leaf. foreach (string name_1 in queueConf.GetConfiguredQueues()[FSQueueType.Parent]) { if (RemoveEmptyIncompatibleQueues(name_1, FSQueueType.Parent)) { GetParentQueue(name_1, true); } } foreach (FSQueue queue in queues.Values) { // Update queue metrics FSQueueMetrics queueMetrics = queue.GetMetrics(); queueMetrics.SetMinShare(queue.GetMinShare()); queueMetrics.SetMaxShare(queue.GetMaxShare()); // Set scheduling policies try { SchedulingPolicy policy = queueConf.GetSchedulingPolicy(queue.GetName()); policy.Initialize(scheduler.GetClusterResource()); queue.SetPolicy(policy); } catch (AllocationConfigurationException ex) { Log.Warn("Cannot apply configured scheduling policy to queue " + queue.GetName(), ex); } } // Update steady fair shares for all queues rootQueue.RecomputeSteadyShares(); // Update the fair share preemption timeouts and preemption for all queues // recursively rootQueue.UpdatePreemptionVariables(); }