/// <exception cref="System.IO.IOException"/> public virtual void TestQueueParsingWithMoveQueue() { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); csConf.SetQueues("root", new string[] { "a" }); csConf.SetQueues("root.a", new string[] { "x", "y" }); csConf.SetCapacity("root.a", 100); csConf.SetCapacity("root.a.x", 50); csConf.SetCapacity("root.a.y", 50); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); csConf.SetQueues("root", new string[] { "a", "x" }); csConf.SetQueues("root.a", new string[] { "y" }); csConf.SetCapacity("root.x", 50); csConf.SetCapacity("root.a", 50); csConf.SetCapacity("root.a.y", 100); capacityScheduler.Reinitialize(csConf, rmContext); }
public virtual void TestQueueParsing() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); ServiceOperations.StopQuietly(capacityScheduler); }
public virtual void TestNestedQueueParsingShouldTrimSpaces() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(TestUtils.GetMockRMContext()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext()); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.IsNotNull(a); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue c = capacityScheduler.GetQueue("c"); NUnit.Framework.Assert.IsNotNull(c); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta); CSQueue a1 = capacityScheduler.GetQueue("a1"); NUnit.Framework.Assert.IsNotNull(a1); NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta); CSQueue a2 = capacityScheduler.GetQueue("a2"); NUnit.Framework.Assert.IsNotNull(a2); NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta); }
/// <exception cref="System.Exception"/> public virtual void TestMaxCapacity() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b", "c" } ); string A = CapacitySchedulerConfiguration.Root + ".a"; conf.SetCapacity(A, 50); conf.SetMaximumCapacity(A, 60); string B = CapacitySchedulerConfiguration.Root + ".b"; conf.SetCapacity(B, 50); conf.SetMaximumCapacity(B, 45); // Should throw an exception bool fail = false; CapacityScheduler capacityScheduler; try { capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(new YarnConfiguration()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, null); } catch (ArgumentException) { fail = true; } NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong maxCap" , fail); conf.SetMaximumCapacity(B, 60); // Now this should work capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(new YarnConfiguration()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, null); fail = false; try { LeafQueue a = (LeafQueue)capacityScheduler.GetQueue(A); a.SetMaxCapacity(45); } catch (ArgumentException) { fail = true; } NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong " + "setMaxCap", fail); capacityScheduler.Stop(); }
public virtual void TestReinitialize() { ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); CapacityScheduler capScheduler = null; try { capScheduler = testUtil.MockCapacityScheduler(10); } catch (IOException e) { NUnit.Framework.Assert.Fail(e.Message); } CapacityReservationSystem reservationSystem = new CapacityReservationSystem(); CapacitySchedulerConfiguration conf = capScheduler.GetConfiguration(); RMContext mockContext = capScheduler.GetRMContext(); reservationSystem.SetRMContext(mockContext); try { reservationSystem.Reinitialize(capScheduler.GetConfiguration(), mockContext); } catch (YarnException e) { NUnit.Framework.Assert.Fail(e.Message); } // Assert queue in original config string planQName = testUtil.GetreservationQueueName(); ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName); // Dynamically add a plan string newQ = "reservation"; NUnit.Framework.Assert.IsNull(reservationSystem.GetPlan(newQ)); testUtil.UpdateQueueConfiguration(conf, newQ); try { capScheduler.Reinitialize(conf, mockContext); } catch (IOException e) { NUnit.Framework.Assert.Fail(e.Message); } try { reservationSystem.Reinitialize(conf, mockContext); } catch (YarnException e) { NUnit.Framework.Assert.Fail(e.Message); } ReservationSystemTestUtil.ValidateNewReservationQueue(reservationSystem, newQ); }
/// <exception cref="System.Exception"/> public virtual void TestRootQueueParsing() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); // non-100 percent value will throw IllegalArgumentException conf.SetCapacity(CapacitySchedulerConfiguration.Root, 90); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(new YarnConfiguration()); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, null); ServiceOperations.StopQuietly(capacityScheduler); }
public virtual void TestQueueParsingWithUnusedLabels() { ImmutableSet <string> labels = ImmutableSet.Of("red", "blue"); // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail nodeLabelManager.AddToCluserNodeLabels(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, rmContext); // check root queue's capacity by label -- they should be all zero CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"), Delta); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); capacityScheduler.Stop(); }
/// <exception cref="System.IO.IOException"/> private void CheckInvalidQMapping(YarnConfiguration conf, CapacityScheduler cs, string mapping, string reason) { bool fail = false; try { conf.Set(CapacitySchedulerConfiguration.QueueMapping, mapping); cs.Reinitialize(conf, null); } catch (IOException) { fail = true; } NUnit.Framework.Assert.IsTrue("invalid mapping did not throw exception for " + reason , fail); }
public virtual void TestRefreshQueuesWithReservations() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a1); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a2); cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f)); // Verify all allocations match tcs.CheckQueueCapacities(cs, ACapacity, BCapacity); // Reinitialize and verify all dynamic queued survived CapacitySchedulerConfiguration conf = cs.GetConfiguration(); conf.SetCapacity(A, 80f); conf.SetCapacity(B, 20f); cs.Reinitialize(conf, rm.GetRMContext()); tcs.CheckQueueCapacities(cs, 80f, 20f); }
public virtual void TestQueueParsingReinitializeWithLabels() { nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfigurationWithoutLabels(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM (), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(conf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(conf); capacityScheduler.Start(); csConf = new CapacitySchedulerConfiguration(); SetupQueueConfigurationWithLabels(csConf); conf = new YarnConfiguration(csConf); capacityScheduler.Reinitialize(conf, rmContext); CheckQueueLabels(capacityScheduler); ServiceOperations.StopQuietly(capacityScheduler); }
/// <exception cref="System.Exception"/> public virtual void TestQueueMapping() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); RMContext rmContext = TestUtils.GetMockRMContext(); cs.SetConf(conf); cs.SetRMContext(rmContext); cs.Init(conf); cs.Start(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping ), typeof(GroupMappingServiceProvider)); conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true"); // configuration parsing tests - negative test cases CheckInvalidQMapping(conf, cs, "x:a:b", "invalid specifier"); CheckInvalidQMapping(conf, cs, "u:a", "no queue specified"); CheckInvalidQMapping(conf, cs, "g:a", "no queue specified"); CheckInvalidQMapping(conf, cs, "u:a:b,g:a", "multiple mappings with invalid mapping" ); CheckInvalidQMapping(conf, cs, "u:a:b,g:a:d:e", "too many path segments"); CheckInvalidQMapping(conf, cs, "u::", "empty source and queue"); CheckInvalidQMapping(conf, cs, "u:", "missing source missing queue"); CheckInvalidQMapping(conf, cs, "u:a:", "empty source missing q"); // simple base case for mapping user to queue conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:a:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // group mapping test conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:agroup:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // %user tests conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:"******"a", Q2, cs); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%user"); cs.Reinitialize(conf, null); CheckQMapping("a", "a", cs); // %primary_group tests conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%primary_group"); cs.Reinitialize(conf, null); CheckQMapping("a", "agroup", cs); // non-primary group mapping conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:asubgroup1:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // space trimming conf.Set(CapacitySchedulerConfiguration.QueueMapping, " u : a : " + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); csConf = new CapacitySchedulerConfiguration(); csConf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); SetupQueueConfiguration(csConf); conf = new YarnConfiguration(csConf); resourceManager = new MockRM(csConf); resourceManager.Start(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping ), typeof(GroupMappingServiceProvider)); conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true"); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q1); // toggle admin override and retry conf.SetBoolean(CapacitySchedulerConfiguration.EnableQueueMappingOverride, false); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q2); // ensure that if a user does not specify a Q, the user mapping is used CheckAppQueue(resourceManager, "user", null, Q1); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:usergroup:" + Q2); SetupQueueConfiguration(csConf); resourceManager.GetResourceScheduler().Reinitialize(conf, null); // ensure that if a user does not specify a Q, the group mapping is used CheckAppQueue(resourceManager, "user", null, Q2); // if the mapping specifies a queue that does not exist, the job is rejected conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:non_existent_queue" ); SetupQueueConfiguration(csConf); bool fail = false; try { resourceManager.GetResourceScheduler().Reinitialize(conf, null); } catch (IOException) { fail = true; } NUnit.Framework.Assert.IsTrue("queue initialization failed for non-existent q", fail ); resourceManager.Stop(); }