/// <summary> /// Get a mock /// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/> /// for use in test cases. /// </summary> /// <returns> /// a mock /// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.RMContext"/> /// for use in test cases /// </returns> public static RMContext GetMockRMContext() { // Null dispatcher Dispatcher nullDispatcher = new _Dispatcher_73(); // No op ContainerAllocationExpirer cae = new ContainerAllocationExpirer(nullDispatcher); Configuration conf = new Configuration(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null , new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf ), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()); RMNodeLabelsManager nlm = Org.Mockito.Mockito.Mock <RMNodeLabelsManager>(); Org.Mockito.Mockito.When(nlm.GetQueueResource(Matchers.Any <string>(), Matchers.Any <Set>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_105()); Org.Mockito.Mockito.When(nlm.GetResourceByLabel(Matchers.Any <string>(), Matchers.Any <Resource>())).ThenAnswer(new _Answer_114()); rmContext.SetNodeLabelManager(nlm); rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >()); return(rmContext); }
/// <summary> /// Test init a queue configuration, children's capacity for a given label /// doesn't equals to 100%. /// </summary> /// <remarks> /// Test init a queue configuration, children's capacity for a given label /// doesn't equals to 100%. This expect IllegalArgumentException thrown. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void TestQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard () { nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); SetupQueueConfigurationWithLabels(csConf); csConf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root + ".b.b3", "red", 24 ); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, ImmutableSet. Of(RMNodeLabelsManager.Any)); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root + ".b", ImmutableSet .Of(RMNodeLabelsManager.Any)); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); ServiceOperations.StopQuietly(capacityScheduler); }
/// <exception cref="System.IO.IOException"/> public virtual void TestQueueParsingWithMoveQueue() { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); csConf.SetQueues("root", new string[] { "a" }); csConf.SetQueues("root.a", new string[] { "x", "y" }); csConf.SetCapacity("root.a", 100); csConf.SetCapacity("root.a.x", 50); csConf.SetCapacity("root.a.y", 50); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); csConf.SetQueues("root", new string[] { "a", "x" }); csConf.SetQueues("root.a", new string[] { "y" }); csConf.SetCapacity("root.x", 50); csConf.SetCapacity("root.a", 50); csConf.SetCapacity("root.a.y", 100); capacityScheduler.Reinitialize(csConf, rmContext); }
public virtual void TestQueueParsingWithUnusedLabels() { ImmutableSet <string> labels = ImmutableSet.Of("red", "blue"); // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail nodeLabelManager.AddToCluserNodeLabels(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.SetConf(conf); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(conf); capacityScheduler.Start(); capacityScheduler.Reinitialize(conf, rmContext); // check root queue's capacity by label -- they should be all zero CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"), Delta); NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"), Delta); CSQueue a = capacityScheduler.GetQueue("a"); NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta); CSQueue b1 = capacityScheduler.GetQueue("b1"); NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta); NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity (), Delta); CSQueue c12 = capacityScheduler.GetQueue("c12"); NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta ); NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity( ), Delta); capacityScheduler.Stop(); }
/// <exception cref="System.IO.IOException"/> public static CapacityScheduler MockCapacityScheduler() { // stolen from TestCapacityScheduler CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); cs.SetConf(new YarnConfiguration()); RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager (conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM (), null); rmContext.SetNodeLabelManager(new NullRMNodeLabelsManager()); cs.SetRMContext(rmContext); cs.Init(conf); return(cs); }
public virtual void TestQueueParsingWithLabelsInherit() { nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); SetupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); CheckQueueLabelsInheritConfig(capacityScheduler); ServiceOperations.StopQuietly(capacityScheduler); }
/// <summary> /// Test init a queue configuration, children's capacity for a given label /// doesn't equals to 100%. /// </summary> /// <remarks> /// Test init a queue configuration, children's capacity for a given label /// doesn't equals to 100%. This expect IllegalArgumentException thrown. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void TestQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent () { nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); SetupQueueConfiguration(csConf); csConf.SetCapacity(CapacitySchedulerConfiguration.Root + ".c.c2", 5); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.SetNodeLabelManager(nodeLabelManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); ServiceOperations.StopQuietly(capacityScheduler); }
public virtual void TestQueueParsingWhenLabelsNotExist() { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); SetupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.Init(conf); nodeLabelsManager.Start(); rmContext.SetNodeLabelManager(nodeLabelsManager); capacityScheduler.SetConf(csConf); capacityScheduler.SetRMContext(rmContext); capacityScheduler.Init(csConf); capacityScheduler.Start(); ServiceOperations.StopQuietly(capacityScheduler); ServiceOperations.StopQuietly(nodeLabelsManager); }