public virtual void TestHeadroom()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
.spyHook);
// Manipulate queue 'a'
LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
string host_0 = "host_0";
string rack_0 = "rack_0";
FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
string user_0 = "user_0";
string user_1 = "user_1";
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
RMContext rmContext = TestUtils.GetMockRMContext();
RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
(0, 0);
Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
);
Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Priority priority_1 = TestUtils.CreateMockPriority(1);
// Submit first application with some resource-requests from user_0,
// and check headroom
ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
0);
FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_0, user_0);
IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();
app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_0.UpdateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
(10 * 16 * Gb, 1);
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
// Submit second application from user_0, check headroom
ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
0);
FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_1, user_0);
IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();
app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_1.UpdateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
// no change
// Submit first application from user_1, check for new headroom
ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
0);
FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_1_0, user_1);
IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();
app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_1_0.UpdateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.CreateResource(90 * 16 * Gb);
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
}
public virtual void TestAMResourceLimit()
{
string user_0 = "user_0";
string user_1 = "user_1";
// This uses the default 10% of cluster value for the max am resources
// which are allowed, at 80GB = 8GB for AM's at the queue level. The user
// am limit is 4G initially (based on the queue absolute capacity)
// when there is only 1 user, and drops to 2G (the userlimit) when there
// is a second user
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(80 * Gb, 40);
queue.UpdateClusterResource(clusterResource, new ResourceLimits(clusterResource));
ActiveUsersManager activeUsersManager = Org.Mockito.Mockito.Mock <ActiveUsersManager
>();
Org.Mockito.Mockito.When(queue.GetActiveUsersManager()).ThenReturn(activeUsersManager
);
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(8 * Gb, 1), queue.GetAMResourceLimit());
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(4 * Gb, 1), queue.GetUserAMResourceLimit());
// Two apps for user_0, both start
int ApplicationId = 0;
FiCaSchedulerApp app_0 = GetMockApplication(ApplicationId++, user_0, Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2 * Gb, 1));
queue.SubmitApplicationAttempt(app_0, user_0);
NUnit.Framework.Assert.AreEqual(1, queue.GetNumActiveApplications());
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications());
NUnit.Framework.Assert.AreEqual(1, queue.GetNumActiveApplications(user_0));
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications(user_0));
Org.Mockito.Mockito.When(activeUsersManager.GetNumActiveUsers()).ThenReturn(1);
FiCaSchedulerApp app_1 = GetMockApplication(ApplicationId++, user_0, Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2 * Gb, 1));
queue.SubmitApplicationAttempt(app_1, user_0);
NUnit.Framework.Assert.AreEqual(2, queue.GetNumActiveApplications());
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications());
NUnit.Framework.Assert.AreEqual(2, queue.GetNumActiveApplications(user_0));
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications(user_0));
// AMLimits unchanged
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(8 * Gb, 1), queue.GetAMResourceLimit());
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(4 * Gb, 1), queue.GetUserAMResourceLimit());
// One app for user_1, starts
FiCaSchedulerApp app_2 = GetMockApplication(ApplicationId++, user_1, Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2 * Gb, 1));
queue.SubmitApplicationAttempt(app_2, user_1);
NUnit.Framework.Assert.AreEqual(3, queue.GetNumActiveApplications());
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications());
NUnit.Framework.Assert.AreEqual(1, queue.GetNumActiveApplications(user_1));
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications(user_1));
Org.Mockito.Mockito.When(activeUsersManager.GetNumActiveUsers()).ThenReturn(2);
// Now userAMResourceLimit drops to the queue configured 50% as there is
// another user active
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(8 * Gb, 1), queue.GetAMResourceLimit());
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(2 * Gb, 1), queue.GetUserAMResourceLimit());
// Second user_1 app cannot start
FiCaSchedulerApp app_3 = GetMockApplication(ApplicationId++, user_1, Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2 * Gb, 1));
queue.SubmitApplicationAttempt(app_3, user_1);
NUnit.Framework.Assert.AreEqual(3, queue.GetNumActiveApplications());
NUnit.Framework.Assert.AreEqual(1, queue.GetNumPendingApplications());
NUnit.Framework.Assert.AreEqual(1, queue.GetNumActiveApplications(user_1));
NUnit.Framework.Assert.AreEqual(1, queue.GetNumPendingApplications(user_1));
// Now finish app so another should be activated
queue.FinishApplicationAttempt(app_2, A);
NUnit.Framework.Assert.AreEqual(3, queue.GetNumActiveApplications());
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications());
NUnit.Framework.Assert.AreEqual(1, queue.GetNumActiveApplications(user_1));
NUnit.Framework.Assert.AreEqual(0, queue.GetNumPendingApplications(user_1));
}