public virtual void TestQueueParsingShouldTrimSpaces()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.IsNotNull(a);
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue c = capacityScheduler.GetQueue("c");
NUnit.Framework.Assert.IsNotNull(c);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
}
public virtual void TestQueueParsingReinitializeWithLabels()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfigurationWithoutLabels(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM
(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
csConf = new CapacitySchedulerConfiguration();
SetupQueueConfigurationWithLabels(csConf);
conf = new YarnConfiguration(csConf);
capacityScheduler.Reinitialize(conf, rmContext);
CheckQueueLabels(capacityScheduler);
ServiceOperations.StopQuietly(capacityScheduler);
}
/// <exception cref="System.Exception"/>
public virtual void TestQueueMapping()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler cs = new CapacityScheduler();
RMContext rmContext = TestUtils.GetMockRMContext();
cs.SetConf(conf);
cs.SetRMContext(rmContext);
cs.Init(conf);
cs.Start();
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping
), typeof(GroupMappingServiceProvider));
conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true");
// configuration parsing tests - negative test cases
CheckInvalidQMapping(conf, cs, "x:a:b", "invalid specifier");
CheckInvalidQMapping(conf, cs, "u:a", "no queue specified");
CheckInvalidQMapping(conf, cs, "g:a", "no queue specified");
CheckInvalidQMapping(conf, cs, "u:a:b,g:a", "multiple mappings with invalid mapping"
);
CheckInvalidQMapping(conf, cs, "u:a:b,g:a:d:e", "too many path segments");
CheckInvalidQMapping(conf, cs, "u::", "empty source and queue");
CheckInvalidQMapping(conf, cs, "u:", "missing source missing queue");
CheckInvalidQMapping(conf, cs, "u:a:", "empty source missing q");
// simple base case for mapping user to queue
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:a:" + Q1);
cs.Reinitialize(conf, null);
CheckQMapping("a", Q1, cs);
// group mapping test
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:agroup:" + Q1);
cs.Reinitialize(conf, null);
CheckQMapping("a", Q1, cs);
// %user tests
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:"******"a", Q2, cs);
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%user");
cs.Reinitialize(conf, null);
CheckQMapping("a", "a", cs);
// %primary_group tests
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%primary_group");
cs.Reinitialize(conf, null);
CheckQMapping("a", "agroup", cs);
// non-primary group mapping
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:asubgroup1:" + Q1);
cs.Reinitialize(conf, null);
CheckQMapping("a", Q1, cs);
// space trimming
conf.Set(CapacitySchedulerConfiguration.QueueMapping, " u : a : " + Q1);
cs.Reinitialize(conf, null);
CheckQMapping("a", Q1, cs);
csConf = new CapacitySchedulerConfiguration();
csConf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
SetupQueueConfiguration(csConf);
conf = new YarnConfiguration(csConf);
resourceManager = new MockRM(csConf);
resourceManager.Start();
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping
), typeof(GroupMappingServiceProvider));
conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true");
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q1);
// toggle admin override and retry
conf.SetBoolean(CapacitySchedulerConfiguration.EnableQueueMappingOverride, false);
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q2);
// ensure that if a user does not specify a Q, the user mapping is used
CheckAppQueue(resourceManager, "user", null, Q1);
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:usergroup:" + Q2);
SetupQueueConfiguration(csConf);
resourceManager.GetResourceScheduler().Reinitialize(conf, null);
// ensure that if a user does not specify a Q, the group mapping is used
CheckAppQueue(resourceManager, "user", null, Q2);
// if the mapping specifies a queue that does not exist, the job is rejected
conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:non_existent_queue"
);
SetupQueueConfiguration(csConf);
bool fail = false;
try
{
resourceManager.GetResourceScheduler().Reinitialize(conf, null);
}
catch (IOException)
{
fail = true;
}
NUnit.Framework.Assert.IsTrue("queue initialization failed for non-existent q", fail
);
resourceManager.Stop();
}