internal CapacitySchedulerQueueInfo(CSQueue q)
{
queuePath = q.GetQueuePath();
capacity = q.GetCapacity() * 100;
usedCapacity = q.GetUsedCapacity() * 100;
maxCapacity = q.GetMaximumCapacity();
if (maxCapacity < Epsilon || maxCapacity > 1f)
{
maxCapacity = 1f;
}
maxCapacity *= 100;
absoluteCapacity = Cap(q.GetAbsoluteCapacity(), 0f, 1f) * 100;
absoluteMaxCapacity = Cap(q.GetAbsoluteMaximumCapacity(), 0f, 1f) * 100;
absoluteUsedCapacity = Cap(q.GetAbsoluteUsedCapacity(), 0f, 1f) * 100;
numApplications = q.GetNumApplications();
queueName = q.GetQueueName();
state = q.GetState();
resourcesUsed = new ResourceInfo(q.GetUsedResources());
if (q is PlanQueue && !((PlanQueue)q).ShowReservationsAsQueues())
{
hideReservationQueues = true;
}
// add labels
ICollection <string> labelSet = q.GetAccessibleNodeLabels();
if (labelSet != null)
{
Sharpen.Collections.AddAll(nodeLabels, labelSet);
nodeLabels.Sort();
}
}
public static void UpdateQueueStatistics(ResourceCalculator calculator, CSQueue childQueue
, CSQueue parentQueue, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource
, Org.Apache.Hadoop.Yarn.Api.Records.Resource minimumAllocation)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource queueLimit = Resources.None();
Org.Apache.Hadoop.Yarn.Api.Records.Resource usedResources = childQueue.GetUsedResources
();
float absoluteUsedCapacity = 0.0f;
float usedCapacity = 0.0f;
if (Resources.GreaterThan(calculator, clusterResource, clusterResource, Resources
.None()))
{
queueLimit = Resources.Multiply(clusterResource, childQueue.GetAbsoluteCapacity()
);
absoluteUsedCapacity = Resources.Divide(calculator, clusterResource, usedResources
, clusterResource);
usedCapacity = Resources.Equals(queueLimit, Resources.None()) ? 0 : Resources.Divide
(calculator, clusterResource, usedResources, queueLimit);
}
childQueue.SetUsedCapacity(usedCapacity);
childQueue.SetAbsoluteUsedCapacity(absoluteUsedCapacity);
Org.Apache.Hadoop.Yarn.Api.Records.Resource available = Resources.Subtract(queueLimit
, usedResources);
childQueue.GetMetrics().SetAvailableResourcesToQueue(Resources.Max(calculator, clusterResource
, available, Resources.None()));
}
public virtual void TestQueueParsing()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
ServiceOperations.StopQuietly(capacityScheduler);
}
public virtual void TestNestedQueueParsingShouldTrimSpaces()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.IsNotNull(a);
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue c = capacityScheduler.GetQueue("c");
NUnit.Framework.Assert.IsNotNull(c);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a1 = capacityScheduler.GetQueue("a1");
NUnit.Framework.Assert.IsNotNull(a1);
NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.IsNotNull(a2);
NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta);
}
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetReservationQueueResourceIfExists
(Plan plan, ReservationId reservationId)
{
CSQueue resQueue = cs.GetQueue(reservationId.ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Resource reservationResource = null;
if (resQueue != null)
{
reservationResource = Resources.Multiply(cs.GetClusterResource(), resQueue.GetAbsoluteCapacity
());
}
return(reservationResource);
}
public virtual void TestQueueParsingWithUnusedLabels()
{
ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");
// Initialize a cluster with labels, but doesn't use them, reinitialize
// shouldn't fail
nodeLabelManager.AddToCluserNodeLabels(labels);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, rmContext);
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
Delta);
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
capacityScheduler.Stop();
}
private void CheckQueueLabels(CapacityScheduler capacityScheduler)
{
// queue-A is red, blue
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A1 inherits A's configuration
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A2 is "red"
NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Count);
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Contains("red"));
// queue-B is "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-B2 inherits "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b2").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// check capacity of A2
CSQueue qA2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.AreEqual(0.7, qA2.GetCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.5, qA2.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0.07, qA2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.25, qA2.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.1275, qA2.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.3, qA2.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
// check capacity of B3
CSQueue qB3 = capacityScheduler.GetQueue("b3");
NUnit.Framework.Assert.AreEqual(0.18, qB3.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.125, qB3.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.35, qB3.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(1, qB3.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
}
private float ComputeQueueUsedCapacity(CSQueue queue, int expectedMemory, Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource)
{
return(expectedMemory / (clusterResource.GetMemory() * queue.GetAbsoluteCapacity
()));
}
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetPlanQueueCapacity
(string planQueueName)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource minAllocation = GetMinAllocation();
ResourceCalculator rescCalc = GetResourceCalculator();
CSQueue planQueue = capScheduler.GetQueue(planQueueName);
return(rescCalc.MultiplyAndNormalizeDown(capScheduler.GetClusterResource(), planQueue
.GetAbsoluteCapacity(), minAllocation));
}
