internal CapacitySchedulerQueueInfo(CSQueue q)
{
queuePath = q.GetQueuePath();
capacity = q.GetCapacity() * 100;
usedCapacity = q.GetUsedCapacity() * 100;
maxCapacity = q.GetMaximumCapacity();
if (maxCapacity < Epsilon || maxCapacity > 1f)
{
maxCapacity = 1f;
}
maxCapacity *= 100;
absoluteCapacity = Cap(q.GetAbsoluteCapacity(), 0f, 1f) * 100;
absoluteMaxCapacity = Cap(q.GetAbsoluteMaximumCapacity(), 0f, 1f) * 100;
absoluteUsedCapacity = Cap(q.GetAbsoluteUsedCapacity(), 0f, 1f) * 100;
numApplications = q.GetNumApplications();
queueName = q.GetQueueName();
state = q.GetState();
resourcesUsed = new ResourceInfo(q.GetUsedResources());
if (q is PlanQueue && !((PlanQueue)q).ShowReservationsAsQueues())
{
hideReservationQueues = true;
}
// add labels
ICollection <string> labelSet = q.GetAccessibleNodeLabels();
if (labelSet != null)
{
Sharpen.Collections.AddAll(nodeLabels, labelSet);
nodeLabels.Sort();
}
}
public virtual void TestNestedQueueParsingShouldTrimSpaces()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.IsNotNull(a);
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue c = capacityScheduler.GetQueue("c");
NUnit.Framework.Assert.IsNotNull(c);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a1 = capacityScheduler.GetQueue("a1");
NUnit.Framework.Assert.IsNotNull(a1);
NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.IsNotNull(a2);
NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta);
}
public virtual void TestQueueParsing()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
ServiceOperations.StopQuietly(capacityScheduler);
}
public static float ComputeAbsoluteMaximumCapacity(float maximumCapacity, CSQueue
parent)
{
float parentAbsMaxCapacity = (parent == null) ? 1.0f : parent.GetAbsoluteMaximumCapacity
();
return(parentAbsMaxCapacity * maximumCapacity);
}
public virtual void TestQueueParsingWithUnusedLabels()
{
ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");
// Initialize a cluster with labels, but doesn't use them, reinitialize
// shouldn't fail
nodeLabelManager.AddToCluserNodeLabels(labels);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, rmContext);
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
Delta);
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
capacityScheduler.Stop();
}
private void CheckQueueLabels(CapacityScheduler capacityScheduler)
{
// queue-A is red, blue
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A1 inherits A's configuration
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A2 is "red"
NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Count);
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Contains("red"));
// queue-B is "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-B2 inherits "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b2").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// check capacity of A2
CSQueue qA2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.AreEqual(0.7, qA2.GetCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.5, qA2.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0.07, qA2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.25, qA2.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.1275, qA2.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.3, qA2.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
// check capacity of B3
CSQueue qB3 = capacityScheduler.GetQueue("b3");
NUnit.Framework.Assert.AreEqual(0.18, qB3.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.125, qB3.GetQueueCapacities().GetAbsoluteCapacity
("red"), Delta);
NUnit.Framework.Assert.AreEqual(0.35, qB3.GetAbsoluteMaximumCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(1, qB3.GetQueueCapacities().GetAbsoluteMaximumCapacity
("red"), Delta);
}
private ResourceLimits GetResourceLimitsOfChild(CSQueue child, Org.Apache.Hadoop.Yarn.Api.Records.Resource
clusterResource, ResourceLimits parentLimits)
{
// Set resource-limit of a given child, child.limit =
// min(my.limit - my.used + child.used, child.max)
// Parent available resource = parent-limit - parent-used-resource
Org.Apache.Hadoop.Yarn.Api.Records.Resource parentMaxAvailableResource = Resources
.Subtract(parentLimits.GetLimit(), GetUsedResources());
// Child's limit = parent-available-resource + child-used
Org.Apache.Hadoop.Yarn.Api.Records.Resource childLimit = Resources.Add(parentMaxAvailableResource
, child.GetUsedResources());
// Get child's max resource
Org.Apache.Hadoop.Yarn.Api.Records.Resource childConfiguredMaxResource = Resources
.MultiplyAndNormalizeDown(resourceCalculator, labelManager.GetResourceByLabel(RMNodeLabelsManager
.NoLabel, clusterResource), child.GetAbsoluteMaximumCapacity(), minimumAllocation
);
// Child's limit should be capped by child configured max resource
childLimit = Resources.Min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource
);
// Normalize before return
childLimit = Resources.RoundDown(resourceCalculator, childLimit, minimumAllocation
);
return(new ResourceLimits(childLimit));
}
