internal CapacitySchedulerQueueInfo(CSQueue q) { queuePath = q.GetQueuePath(); capacity = q.GetCapacity() * 100; usedCapacity = q.GetUsedCapacity() * 100; maxCapacity = q.GetMaximumCapacity(); if (maxCapacity < Epsilon || maxCapacity > 1f) { maxCapacity = 1f; } maxCapacity *= 100; absoluteCapacity = Cap(q.GetAbsoluteCapacity(), 0f, 1f) * 100; absoluteMaxCapacity = Cap(q.GetAbsoluteMaximumCapacity(), 0f, 1f) * 100; absoluteUsedCapacity = Cap(q.GetAbsoluteUsedCapacity(), 0f, 1f) * 100; numApplications = q.GetNumApplications(); queueName = q.GetQueueName(); state = q.GetState(); resourcesUsed = new ResourceInfo(q.GetUsedResources()); if (q is PlanQueue && !((PlanQueue)q).ShowReservationsAsQueues()) { hideReservationQueues = true; } // add labels ICollection <string> labelSet = q.GetAccessibleNodeLabels(); if (labelSet != null) { Sharpen.Collections.AddAll(nodeLabels, labelSet); nodeLabels.Sort(); } }
public static void UpdateQueueStatistics(ResourceCalculator calculator, CSQueue childQueue , CSQueue parentQueue, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource , Org.Apache.Hadoop.Yarn.Api.Records.Resource minimumAllocation) { Org.Apache.Hadoop.Yarn.Api.Records.Resource queueLimit = Resources.None(); Org.Apache.Hadoop.Yarn.Api.Records.Resource usedResources = childQueue.GetUsedResources (); float absoluteUsedCapacity = 0.0f; float usedCapacity = 0.0f; if (Resources.GreaterThan(calculator, clusterResource, clusterResource, Resources .None())) { queueLimit = Resources.Multiply(clusterResource, childQueue.GetAbsoluteCapacity() ); absoluteUsedCapacity = Resources.Divide(calculator, clusterResource, usedResources , clusterResource); usedCapacity = Resources.Equals(queueLimit, Resources.None()) ? 0 : Resources.Divide (calculator, clusterResource, usedResources, queueLimit); } childQueue.SetUsedCapacity(usedCapacity); childQueue.SetAbsoluteUsedCapacity(absoluteUsedCapacity); Org.Apache.Hadoop.Yarn.Api.Records.Resource available = Resources.Subtract(queueLimit , usedResources); childQueue.GetMetrics().SetAvailableResourcesToQueue(Resources.Max(calculator, clusterResource , available, Resources.None())); }
public virtual void TestQueueParsing() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); ServiceOperations.StopQuietly(capacityScheduler); }
public virtual void TestNestedQueueParsingShouldTrimSpaces() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.IsNotNull(a); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue c = capacityScheduler.GetQueue("c"); NUnit.Framework.Assert.IsNotNull(c); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta); CSQueue a1 = capacityScheduler.GetQueue("a1"); NUnit.Framework.Assert.IsNotNull(a1); NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta); CSQueue a2 = capacityScheduler.GetQueue("a2"); NUnit.Framework.Assert.IsNotNull(a2); NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta); }
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetReservationQueueResourceIfExists (Plan plan, ReservationId reservationId) { CSQueue resQueue = cs.GetQueue(reservationId.ToString()); Org.Apache.Hadoop.Yarn.Api.Records.Resource reservationResource = null; if (resQueue != null) { reservationResource = Resources.Multiply(cs.GetClusterResource(), resQueue.GetAbsoluteCapacity ()); } return(reservationResource); }
public virtual void TestQueueParsingWithUnusedLabels() { ImmutableSet <string> labels = ImmutableSet.Of("red", "blue"); // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail nodeLabelManager.AddToCluserNodeLabels(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, rmContext); // check root queue's capacity by label -- they should be all zero CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"), Delta); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); capacityScheduler.Stop(); }
private void CheckQueueLabels(CapacityScheduler capacityScheduler) { // queue-A is red, blue NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A1 inherits A's configuration NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A2 is "red" NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Count); NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Contains("red")); // queue-B is "red"/"blue" NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-B2 inherits "red"/"blue" NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b2").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // check capacity of A2 CSQueue qA2 = capacityScheduler.GetQueue("a2"); NUnit.Framework.Assert.AreEqual(0.7, qA2.GetCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.5, qA2.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0.07, qA2.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.25, qA2.GetQueueCapacities().GetAbsoluteCapacity ("red"), Delta); NUnit.Framework.Assert.AreEqual(0.1275, qA2.GetAbsoluteMaximumCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.3, qA2.GetQueueCapacities().GetAbsoluteMaximumCapacity ("red"), Delta); // check capacity of B3 CSQueue qB3 = capacityScheduler.GetQueue("b3"); NUnit.Framework.Assert.AreEqual(0.18, qB3.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.125, qB3.GetQueueCapacities().GetAbsoluteCapacity ("red"), Delta); NUnit.Framework.Assert.AreEqual(0.35, qB3.GetAbsoluteMaximumCapacity(), Delta); NUnit.Framework.Assert.AreEqual(1, qB3.GetQueueCapacities().GetAbsoluteMaximumCapacity ("red"), Delta); }
private float ComputeQueueUsedCapacity(CSQueue queue, int expectedMemory, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource) { return(expectedMemory / (clusterResource.GetMemory() * queue.GetAbsoluteCapacity ())); }
protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetPlanQueueCapacity (string planQueueName) { Org.Apache.Hadoop.Yarn.Api.Records.Resource minAllocation = GetMinAllocation(); ResourceCalculator rescCalc = GetResourceCalculator(); CSQueue planQueue = capScheduler.GetQueue(planQueueName); return(rescCalc.MultiplyAndNormalizeDown(capScheduler.GetClusterResource(), planQueue .GetAbsoluteCapacity(), minAllocation)); }