public virtual void TestHeadroom()
{
FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>();
Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock(
));
FSLeafQueue mockQueue = Org.Mockito.Mockito.Mock <FSLeafQueue>();
Resource queueMaxResources = Resource.NewInstance(5 * 1024, 3);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource
(4096, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract(
queueFairShare, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources
.Subtract(queueMaxResources, queueUsage);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(8192, 8);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource
(2048, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract
(clusterResource, clusterUsage);
QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources);
Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare);
Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage);
Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource
);
Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn
(clusterUsage);
Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId,
"user1", mockQueue, null, rmContext);
// Min of Memory and CPU across cluster and queue is used in
// DominantResourceFairnessPolicy
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(DominantResourceFairnessPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores
(), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores
()));
// Fair and Fifo ignore CPU of queue, so use cluster available CPU
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FairSharePolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
(typeof(FifoPolicy)));
VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
(), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
(), queueMaxResourcesAvailable.GetVirtualCores()));
}
public virtual QueueInfo GetQueueInfo(bool includeChildQueues, bool recursive)
{
QueueInfo queueInfo = recordFactory.NewRecordInstance <QueueInfo>();
queueInfo.SetQueueName(GetQueueName());
if (scheduler.GetClusterResource().GetMemory() == 0)
{
queueInfo.SetCapacity(0.0f);
}
else
{
queueInfo.SetCapacity((float)GetFairShare().GetMemory() / scheduler.GetClusterResource
().GetMemory());
}
if (GetFairShare().GetMemory() == 0)
{
queueInfo.SetCurrentCapacity(0.0f);
}
else
{
queueInfo.SetCurrentCapacity((float)GetResourceUsage().GetMemory() / GetFairShare
().GetMemory());
}
AList <QueueInfo> childQueueInfos = new AList <QueueInfo>();
if (includeChildQueues)
{
ICollection <Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueue>
childQueues = GetChildQueues();
foreach (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.FSQueue child
in childQueues)
{
childQueueInfos.AddItem(child.GetQueueInfo(recursive, recursive));
}
}
queueInfo.SetChildQueues(childQueueInfos);
queueInfo.SetQueueState(QueueState.Running);
return(queueInfo);
}
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler)
{
AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration();
queueName = queue.GetName();
schedulingPolicy = queue.GetPolicy().GetName();
clusterResources = new ResourceInfo(scheduler.GetClusterResource());
usedResources = new ResourceInfo(queue.GetResourceUsage());
fractionMemUsed = (float)usedResources.GetMemory() / clusterResources.GetMemory();
steadyFairResources = new ResourceInfo(queue.GetSteadyFairShare());
fairResources = new ResourceInfo(queue.GetFairShare());
minResources = new ResourceInfo(queue.GetMinShare());
maxResources = new ResourceInfo(queue.GetMaxShare());
maxResources = new ResourceInfo(Resources.ComponentwiseMin(queue.GetMaxShare(), scheduler
.GetClusterResource()));
fractionMemSteadyFairShare = (float)steadyFairResources.GetMemory() / clusterResources
.GetMemory();
fractionMemFairShare = (float)fairResources.GetMemory() / clusterResources.GetMemory
();
fractionMemMinShare = (float)minResources.GetMemory() / clusterResources.GetMemory
();
fractionMemMaxShare = (float)maxResources.GetMemory() / clusterResources.GetMemory
();
maxApps = allocConf.GetQueueMaxApps(queueName);
childQueues = new AList <Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.Dao.FairSchedulerQueueInfo
>();
if (allocConf.IsReservable(queueName) && !allocConf.GetShowReservationAsQueues(queueName
))
{
return;
}
ICollection <FSQueue> children = queue.GetChildQueues();
foreach (FSQueue child in children)
{
if (child is FSLeafQueue)
{
childQueues.AddItem(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler)
);
}
else
{
childQueues.AddItem(new Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.Dao.FairSchedulerQueueInfo
(child, scheduler));
}
}
}
public virtual void UpdateAllocationConfiguration(AllocationConfiguration queueConf
)
{
// Create leaf queues and the parent queues in a leaf's ancestry if they do not exist
foreach (string name in queueConf.GetConfiguredQueues()[FSQueueType.Leaf])
{
if (RemoveEmptyIncompatibleQueues(name, FSQueueType.Leaf))
{
GetLeafQueue(name, true);
}
}
// At this point all leaves and 'parents with at least one child' would have been created.
// Now create parents with no configured leaf.
foreach (string name_1 in queueConf.GetConfiguredQueues()[FSQueueType.Parent])
{
if (RemoveEmptyIncompatibleQueues(name_1, FSQueueType.Parent))
{
GetParentQueue(name_1, true);
}
}
foreach (FSQueue queue in queues.Values)
{
// Update queue metrics
FSQueueMetrics queueMetrics = queue.GetMetrics();
queueMetrics.SetMinShare(queue.GetMinShare());
queueMetrics.SetMaxShare(queue.GetMaxShare());
// Set scheduling policies
try
{
SchedulingPolicy policy = queueConf.GetSchedulingPolicy(queue.GetName());
policy.Initialize(scheduler.GetClusterResource());
queue.SetPolicy(policy);
}
catch (AllocationConfigurationException ex)
{
Log.Warn("Cannot apply configured scheduling policy to queue " + queue.GetName(),
ex);
}
}
// Update steady fair shares for all queues
rootQueue.RecomputeSteadyShares();
// Update the fair share preemption timeouts and preemption for all queues
// recursively
rootQueue.UpdatePreemptionVariables();
}