public virtual void Setup()
{
// setup a context / conf
csConf = new CapacitySchedulerConfiguration();
YarnConfiguration conf = new YarnConfiguration();
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb, 1));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb, 32));
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
RMContext mockRMContext = TestUtils.GetMockRMContext();
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(mockRMContext);
// create a queue
PlanQueue pq = new PlanQueue(csContext, "root", null, null);
reservationQueue = new ReservationQueue(csContext, "a", pq);
}
public virtual void SetUp()
{
CapacityScheduler spyCs = new CapacityScheduler();
cs = Org.Mockito.Mockito.Spy(spyCs);
scheduler = cs;
rmContext = TestUtils.GetMockRMContext();
spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Org.Mockito.Mockito.When(spyRMContext.GetScheduler()).ThenReturn(scheduler);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
ReservationSystemTestUtil.SetupQueueConfiguration(csConf);
cs.SetConf(csConf);
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(minAlloc
);
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(maxAlloc
);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(scheduler.GetClusterResource()).ThenReturn(Resources.CreateResource
(125 * Gb, 125));
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(new DefaultResourceCalculator
());
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(csConf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
cs.SetRMContext(spyRMContext);
cs.Init(csConf);
cs.Start();
SetupPlanFollower();
}
public virtual void SetUp()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
YarnConfiguration conf = new YarnConfiguration();
SetupQueueConfiguration(csConf);
rmContext = TestUtils.GetMockRMContext();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb, 1));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb, 32));
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(10 * 16 * Gb, 10 * 32));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues
, queues, TestUtils.spyHook);
queue = Org.Mockito.Mockito.Spy(new LeafQueue(csContext, A, root, null));
// Stub out ACL checks
Org.Mockito.Mockito.DoReturn(true).When(queue).HasAccess(Matchers.Any <QueueACL>()
, Matchers.Any <UserGroupInformation>());
// Some default values
Org.Mockito.Mockito.DoReturn(100).When(queue).GetMaxApplications();
Org.Mockito.Mockito.DoReturn(25).When(queue).GetMaxApplicationsPerUser();
}
public virtual void SetUp()
{
rmContext = TestUtils.GetMockRMContext();
conf = new YarnConfiguration();
csConf = new CapacitySchedulerConfiguration();
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb, 1));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb, 32));
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceComparator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
}
/// <exception cref="System.IO.IOException"/>
public AbstractCSQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent
, CSQueue old)
{
// Track resource usage-by-label like used-resource/pending-resource, etc.
// Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
this.labelManager = cs.GetRMContext().GetNodeLabelManager();
this.parent = parent;
this.queueName = queueName;
this.resourceCalculator = cs.GetResourceCalculator();
// must be called after parent and queueName is set
this.metrics = old != null?old.GetMetrics() : QueueMetrics.ForQueue(GetQueuePath
(), parent, cs.GetConfiguration().GetEnableUserMetrics(), cs.GetConf());
this.csContext = cs;
this.minimumAllocation = csContext.GetMinimumResourceCapability();
// initialize ResourceUsage
queueUsage = new ResourceUsage();
queueEntity = new PrivilegedEntity(PrivilegedEntity.EntityType.Queue, GetQueuePath
());
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
}
public virtual void TestHeadroom()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
.spyHook);
// Manipulate queue 'a'
LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
string host_0 = "host_0";
string rack_0 = "rack_0";
FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
string user_0 = "user_0";
string user_1 = "user_1";
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
RMContext rmContext = TestUtils.GetMockRMContext();
RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
(0, 0);
Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
);
Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Priority priority_1 = TestUtils.CreateMockPriority(1);
// Submit first application with some resource-requests from user_0,
// and check headroom
ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
0);
FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_0, user_0);
IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();
app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_0.UpdateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
(10 * 16 * Gb, 1);
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
// Submit second application from user_0, check headroom
ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
0);
FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_1, user_0);
IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();
app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_1.UpdateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
// no change
// Submit first application from user_1, check for new headroom
ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
0);
FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_1_0, user_1);
IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();
app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_1_0.UpdateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.CreateResource(90 * 16 * Gb);
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
}
public virtual void TestLimitsComputation()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb, 1));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb, 16));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb, 100 * 16);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues
, queues, TestUtils.spyHook);
LeafQueue queue = (LeafQueue)queues[A];
Log.Info("Queue 'A' -" + " AMResourceLimit=" + queue.GetAMResourceLimit() + " UserAMResourceLimit="
+ queue.GetUserAMResourceLimit());
NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(160 * Gb, 1));
NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(80 * Gb, 1));
NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity
()), queue.GetMetrics().GetAvailableMB());
// Add some nodes to the cluster & test new limits
clusterResource = Resources.CreateResource(120 * 16 * Gb);
root.UpdateClusterResource(clusterResource, new ResourceLimits(clusterResource));
NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(192 * Gb, 1));
NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(96 * Gb, 1));
NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity
()), queue.GetMetrics().GetAvailableMB());
// should return -1 if per queue setting not set
NUnit.Framework.Assert.AreEqual((int)CapacitySchedulerConfiguration.Undefined, csConf
.GetMaximumApplicationsPerQueue(queue.GetQueuePath()));
int expectedMaxApps = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions
* queue.GetAbsoluteCapacity());
NUnit.Framework.Assert.AreEqual(expectedMaxApps, queue.GetMaxApplications());
int expectedMaxAppsPerUser = (int)(expectedMaxApps * (queue.GetUserLimit() / 100.0f
) * queue.GetUserLimitFactor());
NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser
());
// should default to global setting if per queue setting not set
NUnit.Framework.Assert.AreEqual((long)CapacitySchedulerConfiguration.DefaultMaximumApplicationmastersResourcePercent
, (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent(queue.GetQueuePath
()));
// Change the per-queue max AM resources percentage.
csConf.SetFloat("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-am-resource-percent"
, 0.5f);
// Re-create queues to get new configs.
queues = new Dictionary <string, CSQueue>();
root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues
, TestUtils.spyHook);
clusterResource = Resources.CreateResource(100 * 16 * Gb);
queue = (LeafQueue)queues[A];
NUnit.Framework.Assert.AreEqual((long)0.5, (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent
(queue.GetQueuePath()));
NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(800 * Gb, 1));
NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(400 * Gb, 1));
// Change the per-queue max applications.
csConf.SetInt("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-applications"
, 9999);
// Re-create queues to get new configs.
queues = new Dictionary <string, CSQueue>();
root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues
, TestUtils.spyHook);
queue = (LeafQueue)queues[A];
NUnit.Framework.Assert.AreEqual(9999, (int)csConf.GetMaximumApplicationsPerQueue(
queue.GetQueuePath()));
NUnit.Framework.Assert.AreEqual(9999, queue.GetMaxApplications());
expectedMaxAppsPerUser = (int)(9999 * (queue.GetUserLimit() / 100.0f) * queue.GetUserLimitFactor
());
NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser
());
}