public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
//upload default configurations
UploadDefaultConfiguration();
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler();
int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
UploadConfiguration(csConf, "capacity-scheduler.xml");
rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance());
int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();
NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore);
}
public virtual void TestReinitialize()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler capScheduler = null;
try
{
capScheduler = testUtil.MockCapacityScheduler(10);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
CapacityReservationSystem reservationSystem = new CapacityReservationSystem();
CapacitySchedulerConfiguration conf = capScheduler.GetConfiguration();
RMContext mockContext = capScheduler.GetRMContext();
reservationSystem.SetRMContext(mockContext);
try
{
reservationSystem.Reinitialize(capScheduler.GetConfiguration(), mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
// Assert queue in original config
string planQName = testUtil.GetreservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName);
// Dynamically add a plan
string newQ = "reservation";
NUnit.Framework.Assert.IsNull(reservationSystem.GetPlan(newQ));
testUtil.UpdateQueueConfiguration(conf, newQ);
try
{
capScheduler.Reinitialize(conf, mockContext);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
try
{
reservationSystem.Reinitialize(conf, mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
ReservationSystemTestUtil.ValidateNewReservationQueue(reservationSystem, newQ);
}
public virtual void TestAdminRefreshQueuesWithLocalConfigurationProvider()
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler();
int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications();
try
{
rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance());
NUnit.Framework.Assert.AreEqual(maxAppsBefore, cs.GetConfiguration().GetMaximumSystemApplications
());
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Using localConfigurationProvider. Should not get any exception."
);
}
}
/// <exception cref="System.Exception"/>
private void SetupPlanFollower()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
mClock = Org.Mockito.Mockito.Mock <Clock>();
mAgent = Org.Mockito.Mockito.Mock <ReservationAgent>();
string reservationQ = testUtil.GetFullReservationQueueName();
CapacitySchedulerConfiguration csConf = cs.GetConfiguration();
csConf.SetReservationWindow(reservationQ, 20L);
csConf.SetMaximumCapacity(reservationQ, 40);
csConf.SetAverageCapacity(reservationQ, 20);
policy.Init(reservationQ, csConf);
}
public virtual void TestRefreshQueuesWithReservations()
{
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
// Test add one reservation dynamically and manually modify capacity
ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a"));
cs.AddQueue(a1);
a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f));
// Test add another reservation queue and use setEntitlement to modify
// capacity
ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a"));
cs.AddQueue(a2);
cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f));
// Verify all allocations match
tcs.CheckQueueCapacities(cs, ACapacity, BCapacity);
// Reinitialize and verify all dynamic queued survived
CapacitySchedulerConfiguration conf = cs.GetConfiguration();
conf.SetCapacity(A, 80f);
conf.SetCapacity(B, 20f);
cs.Reinitialize(conf, rm.GetRMContext());
tcs.CheckQueueCapacities(cs, 80f, 20f);
}
protected internal override ReservationSchedulerConfiguration GetReservationSchedulerConfiguration
()
{
return(capScheduler.GetConfiguration());
}
public virtual void TestRMInitialsWithFileSystemBasedConfigurationProvider()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
// upload configurations
FilePath excludeHostsFile = new FilePath(tmpDir.ToString(), "excludeHosts");
if (excludeHostsFile.Exists())
{
excludeHostsFile.Delete();
}
if (!excludeHostsFile.CreateNewFile())
{
NUnit.Framework.Assert.Fail("Can not create " + "excludeHosts");
}
PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
fileWriter.Write("0.0.0.0:123");
fileWriter.Close();
UploadToRemoteFileSystem(new Path(excludeHostsFile.GetAbsolutePath()));
YarnConfiguration yarnConf = new YarnConfiguration();
yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda");
yarnConf.Set(YarnConfiguration.RmNodesExcludeFilePath, this.workingPath + "/excludeHosts"
);
UploadConfiguration(yarnConf, "yarn-site.xml");
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
UploadConfiguration(csConf, "capacity-scheduler.xml");
string aclsString = "alice,bob users,wheel";
Configuration newConf = new Configuration();
newConf.Set("security.applicationclient.protocol.acl", aclsString);
UploadConfiguration(newConf, "hadoop-policy.xml");
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, true);
conf.Set("hadoop.proxyuser.test.groups", "test_groups");
conf.Set("hadoop.proxyuser.test.hosts", "test_hosts");
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
), typeof(GroupMappingServiceProvider));
UploadConfiguration(conf, "core-site.xml");
// update the groups
TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
ResourceManager resourceManager = null;
try
{
try
{
resourceManager = new ResourceManager();
resourceManager.Init(configuration);
resourceManager.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
// validate values for excludeHosts
ICollection <string> excludeHosts = resourceManager.GetRMContext().GetNodesListManager
().GetHostsReader().GetExcludedHosts();
NUnit.Framework.Assert.IsTrue(excludeHosts.Count == 1);
NUnit.Framework.Assert.IsTrue(excludeHosts.Contains("0.0.0.0:123"));
// validate values for admin-acls
string aclStringAfter = resourceManager.adminService.GetAccessControlList().GetAclString
().Trim();
NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation
.GetCurrentUser().GetShortUserName());
// validate values for queue configuration
CapacityScheduler cs = (CapacityScheduler)resourceManager.GetRMContext().GetScheduler
();
int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();
NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
// verify service Acls for AdminService
ServiceAuthorizationManager adminServiceServiceManager = resourceManager.adminService
.GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(adminServiceServiceManager, typeof(ApplicationClientProtocolPB
), aclsString);
// verify service ACLs for ClientRMService
ServiceAuthorizationManager clientRMServiceServiceManager = resourceManager.GetRMContext
().GetClientRMService().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(clientRMServiceServiceManager, typeof(ApplicationClientProtocolPB
), aclsString);
// verify service ACLs for ApplicationMasterService
ServiceAuthorizationManager appMasterService = resourceManager.GetRMContext().GetApplicationMasterService
().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(appMasterService, typeof(ApplicationClientProtocolPB), aclsString
);
// verify service ACLs for ResourceTrackerService
ServiceAuthorizationManager RTService = resourceManager.GetRMContext().GetResourceTrackerService
().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(RTService, typeof(ApplicationClientProtocolPB), aclsString
);
// verify ProxyUsers and ProxyHosts
ProxyUsers.RefreshSuperUserGroupsConfiguration(configuration);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Contains("test_groups"));
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Contains("test_hosts"));
// verify UserToGroupsMappings
IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
(UserGroupInformation.GetCurrentUser().GetUserName());
NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
);
}
finally
{
if (resourceManager != null)
{
resourceManager.Stop();
}
}
}