public virtual void TestQueueParsing() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); ServiceOperations.StopQuietly(capacityScheduler); }
public virtual void TestNestedQueueParsingShouldTrimSpaces() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.IsNotNull(a); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue c = capacityScheduler.GetQueue("c"); NUnit.Framework.Assert.IsNotNull(c); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta); CSQueue a1 = capacityScheduler.GetQueue("a1"); NUnit.Framework.Assert.IsNotNull(a1); NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta); CSQueue a2 = capacityScheduler.GetQueue("a2"); NUnit.Framework.Assert.IsNotNull(a2); NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta); }
protected internal override Queue GetPlanQueue(string planQueueName) { CSQueue queue = cs.GetQueue(planQueueName); if (!(queue is PlanQueue)) { Log.Error("The Plan is not an PlanQueue!"); return(null); } return(queue); }
public virtual void TestQueueParsingWithUnusedLabels() { ImmutableSet <string> labels = ImmutableSet.Of("red", "blue"); // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail nodeLabelManager.AddToCluserNodeLabels(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, rmContext); // check root queue's capacity by label -- they should be all zero CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"), Delta); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); capacityScheduler.Stop(); }
private void CheckQueueLabelsInheritConfig(CapacityScheduler capacityScheduler) { // queue-A is red, blue NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A1 inherits A's configuration NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A2 is "red" NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Count); NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Contains("red")); // queue-B is "red"/"blue" NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels ().IsEmpty()); }
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label ) { CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler(); CSQueue queue = scheduler.GetQueue(queueName); NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label ).GetMemory()); }
public virtual void TestAddQueueFailCases() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); try { // Test invalid addition (adding non-zero size queue) ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); cs.AddQueue(a1); NUnit.Framework.Assert.Fail(); } catch (Exception) { } // expected // Test add one reservation dynamically and manually modify capacity ReservationQueue a1_1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a" )); cs.AddQueue(a1_1); a1_1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a2); try { // Test invalid entitlement (sum of queues exceed 100%) cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100 + 0.1f, 1.0f)); NUnit.Framework.Assert.Fail(); } catch (Exception) { } // expected cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f)); // Verify all allocations match tcs.CheckQueueCapacities(cs, ACapacity, BCapacity); cs.Stop(); }
public virtual void TestRemoveQueue() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a1); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // submit an app RMApp app = rm.SubmitApp(Gb, "test-move-1", "user_0", null, "a1"); // check preconditions IList <ApplicationAttemptId> appsInA1 = cs.GetAppsInQueue("a1"); NUnit.Framework.Assert.AreEqual(1, appsInA1.Count); try { cs.RemoveQueue("a1"); NUnit.Framework.Assert.Fail(); } catch (SchedulerDynamicEditException) { } // expected a1 contains applications // clear queue by killling all apps cs.KillAllAppsInQueue("a1"); // wait for events of move to propagate rm.WaitForState(app.GetApplicationId(), RMAppState.Killed); try { cs.RemoveQueue("a1"); NUnit.Framework.Assert.Fail(); } catch (SchedulerDynamicEditException) { } // expected a1 is not zero capacity // set capacity to zero cs.SetEntitlement("a1", new QueueEntitlement(0f, 0f)); cs.RemoveQueue("a1"); NUnit.Framework.Assert.IsTrue(cs.GetQueue("a1") == null); rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestMaxCapacity() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b", "c" } ); string A = CapacitySchedulerConfiguration.Root + ".a"; conf.SetCapacity(A, 50); conf.SetMaximumCapacity(A, 60); string B = CapacitySchedulerConfiguration.Root + ".b"; conf.SetCapacity(B, 50); conf.SetMaximumCapacity(B, 45); // Should throw an exception bool fail = false; CapacityScheduler capacityScheduler; try { capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(new YarnConfiguration()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, null); } catch (ArgumentException) { fail = true; } NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong maxCap" , fail); conf.SetMaximumCapacity(B, 60); // Now this should work capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(new YarnConfiguration()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, null); fail = false; try { LeafQueue a = (LeafQueue)capacityScheduler.GetQueue(A); a.SetMaxCapacity(45); } catch (ArgumentException) { fail = true; } NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong " + "setMaxCap", fail); capacityScheduler.Stop(); }
public virtual void TestRefreshQueuesWithReservations() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a1); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a2); cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f)); // Verify all allocations match tcs.CheckQueueCapacities(cs, ACapacity, BCapacity); // Reinitialize and verify all dynamic queued survived CapacitySchedulerConfiguration conf = cs.GetConfiguration(); conf.SetCapacity(A, 80f); conf.SetCapacity(B, 20f); cs.Reinitialize(conf, rm.GetRMContext()); tcs.CheckQueueCapacities(cs, 80f, 20f); }
private void CheckQueueLabels(CapacityScheduler capacityScheduler) { // queue-A is red, blue NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A1 inherits A's configuration NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-A2 is "red" NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Count); NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels ().Contains("red")); // queue-B is "red"/"blue" NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // queue-B2 inherits "red"/"blue" NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b2").GetAccessibleNodeLabels ().ContainsAll(ImmutableSet.Of("red", "blue"))); // check capacity of A2 CSQueue qA2 = capacityScheduler.GetQueue("a2"); NUnit.Framework.Assert.AreEqual(0.7, qA2.GetCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.5, qA2.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0.07, qA2.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.25, qA2.GetQueueCapacities().GetAbsoluteCapacity ("red"), Delta); NUnit.Framework.Assert.AreEqual(0.1275, qA2.GetAbsoluteMaximumCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.3, qA2.GetQueueCapacities().GetAbsoluteMaximumCapacity ("red"), Delta); // check capacity of B3 CSQueue qB3 = capacityScheduler.GetQueue("b3"); NUnit.Framework.Assert.AreEqual(0.18, qB3.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.125, qB3.GetQueueCapacities().GetAbsoluteCapacity ("red"), Delta); NUnit.Framework.Assert.AreEqual(0.35, qB3.GetAbsoluteMaximumCapacity(), Delta); NUnit.Framework.Assert.AreEqual(1, qB3.GetQueueCapacities().GetAbsoluteMaximumCapacity ("red"), Delta); }
protected internal override Queue GetDefaultQueue() { return(cs.GetQueue("dedicated" + ReservationConstants.DefaultQueueSuffix)); }
public virtual void TestMoveAppToPlanQueue() { CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler(); // submit an app RMApp app = rm.SubmitApp(Gb, "test-move-1", "user_0", null, "b1"); ApplicationAttemptId appAttemptId = rm.GetApplicationReport(app.GetApplicationId( )).GetCurrentApplicationAttemptId(); // check preconditions IList <ApplicationAttemptId> appsInB1 = scheduler.GetAppsInQueue("b1"); NUnit.Framework.Assert.AreEqual(1, appsInB1.Count); IList <ApplicationAttemptId> appsInB = scheduler.GetAppsInQueue("b"); NUnit.Framework.Assert.AreEqual(1, appsInB.Count); NUnit.Framework.Assert.IsTrue(appsInB.Contains(appAttemptId)); IList <ApplicationAttemptId> appsInA = scheduler.GetAppsInQueue("a"); NUnit.Framework.Assert.IsTrue(appsInA.IsEmpty()); string queue = scheduler.GetApplicationAttempt(appsInB1[0]).GetQueue().GetQueueName (); NUnit.Framework.Assert.IsTrue(queue.Equals("b1")); IList <ApplicationAttemptId> appsInRoot = scheduler.GetAppsInQueue("root"); NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId)); NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count); // create the default reservation queue string defQName = "a" + ReservationConstants.DefaultQueueSuffix; ReservationQueue defQ = new ReservationQueue(scheduler, defQName, (PlanQueue)scheduler .GetQueue("a")); scheduler.AddQueue(defQ); defQ.SetEntitlement(new QueueEntitlement(1f, 1f)); IList <ApplicationAttemptId> appsInDefQ = scheduler.GetAppsInQueue(defQName); NUnit.Framework.Assert.IsTrue(appsInDefQ.IsEmpty()); // now move the app to plan queue scheduler.MoveApplication(app.GetApplicationId(), "a"); // check postconditions appsInDefQ = scheduler.GetAppsInQueue(defQName); NUnit.Framework.Assert.AreEqual(1, appsInDefQ.Count); queue = scheduler.GetApplicationAttempt(appsInDefQ[0]).GetQueue().GetQueueName(); NUnit.Framework.Assert.IsTrue(queue.Equals(defQName)); appsInA = scheduler.GetAppsInQueue("a"); NUnit.Framework.Assert.IsTrue(appsInA.Contains(appAttemptId)); NUnit.Framework.Assert.AreEqual(1, appsInA.Count); appsInRoot = scheduler.GetAppsInQueue("root"); NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId)); NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count); appsInB1 = scheduler.GetAppsInQueue("b1"); NUnit.Framework.Assert.IsTrue(appsInB1.IsEmpty()); appsInB = scheduler.GetAppsInQueue("b"); NUnit.Framework.Assert.IsTrue(appsInB.IsEmpty()); rm.Stop(); }
protected internal override string GetPlanQueuePath(string planQueueName) { return(capScheduler.GetQueue(planQueueName).GetQueuePath()); }