public virtual void Setup() { // setup a context / conf csConf = new CapacitySchedulerConfiguration(); YarnConfiguration conf = new YarnConfiguration(); csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb, 1)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb, 32)); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource (100 * 16 * Gb, 100 * 32)); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); RMContext mockRMContext = TestUtils.GetMockRMContext(); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(mockRMContext); // create a queue PlanQueue pq = new PlanQueue(csContext, "root", null, null); reservationQueue = new ReservationQueue(csContext, "a", pq); }
public virtual void SetUp() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); YarnConfiguration conf = new YarnConfiguration(); SetupQueueConfiguration(csConf); rmContext = TestUtils.GetMockRMContext(); CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext >(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb, 1)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb, 32)); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource (10 * 16 * Gb, 10 * 32)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.RollMasterKey(); Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager ); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues , queues, TestUtils.spyHook); queue = Org.Mockito.Mockito.Spy(new LeafQueue(csContext, A, root, null)); // Stub out ACL checks Org.Mockito.Mockito.DoReturn(true).When(queue).HasAccess(Matchers.Any <QueueACL>() , Matchers.Any <UserGroupInformation>()); // Some default values Org.Mockito.Mockito.DoReturn(100).When(queue).GetMaxApplications(); Org.Mockito.Mockito.DoReturn(25).When(queue).GetMaxApplicationsPerUser(); }
public virtual void SetUp() { rmContext = TestUtils.GetMockRMContext(); conf = new YarnConfiguration(); csConf = new CapacitySchedulerConfiguration(); csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>(); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb, 1)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb, 32)); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource (100 * 16 * Gb, 100 * 32)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceComparator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); }
/// <exception cref="System.IO.IOException"/> public AbstractCSQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent , CSQueue old) { // Track resource usage-by-label like used-resource/pending-resource, etc. // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. this.labelManager = cs.GetRMContext().GetNodeLabelManager(); this.parent = parent; this.queueName = queueName; this.resourceCalculator = cs.GetResourceCalculator(); // must be called after parent and queueName is set this.metrics = old != null?old.GetMetrics() : QueueMetrics.ForQueue(GetQueuePath (), parent, cs.GetConfiguration().GetEnableUserMetrics(), cs.GetConf()); this.csContext = cs; this.minimumAllocation = csContext.GetMinimumResourceCapability(); // initialize ResourceUsage queueUsage = new ResourceUsage(); queueEntity = new PrivilegedEntity(PrivilegedEntity.EntityType.Queue, GetQueuePath ()); // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); }
public virtual void TestHeadroom() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext >(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); // Say cluster has 100 nodes of 16G each Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (100 * 16 * Gb); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource ); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils .spyHook); // Manipulate queue 'a' LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]); string host_0 = "host_0"; string rack_0 = "rack_0"; FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb); string user_0 = "user_0"; string user_1 = "user_1"; RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null); RMContext rmContext = TestUtils.GetMockRMContext(); RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext); ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap <ApplicationId, RMApp>()); RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>(); ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>(); Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource (0, 0); Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource ); Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest ); Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()]; Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps); Priority priority_1 = TestUtils.CreateMockPriority(1); // Submit first application with some resource-requests from user_0, // and check headroom ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_0_0, user_0); IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>(); app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_0_0.UpdateResourceRequests(app_0_0_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource (10 * 16 * Gb, 1); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); // Submit second application from user_0, check headroom ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_0_1, user_0); IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>(); app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_0_1.UpdateResourceRequests(app_0_1_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); // no change // Submit first application from user_1, check for new headroom ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_1_0, user_1); IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>(); app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_1_0.UpdateResourceRequests(app_1_0_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1); // changes NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom()); // Now reduce cluster size and check for the smaller headroom clusterResource = Resources.CreateResource(90 * 16 * Gb); queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1); // changes NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom()); }
public virtual void TestLimitsComputation() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext >(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb, 1)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb, 16)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); // Say cluster has 100 nodes of 16G each Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (100 * 16 * Gb, 100 * 16); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource ); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues , queues, TestUtils.spyHook); LeafQueue queue = (LeafQueue)queues[A]; Log.Info("Queue 'A' -" + " AMResourceLimit=" + queue.GetAMResourceLimit() + " UserAMResourceLimit=" + queue.GetUserAMResourceLimit()); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(160 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(80 * Gb, 1)); NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity ()), queue.GetMetrics().GetAvailableMB()); // Add some nodes to the cluster & test new limits clusterResource = Resources.CreateResource(120 * 16 * Gb); root.UpdateClusterResource(clusterResource, new ResourceLimits(clusterResource)); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(192 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(96 * Gb, 1)); NUnit.Framework.Assert.AreEqual((int)(clusterResource.GetMemory() * queue.GetAbsoluteCapacity ()), queue.GetMetrics().GetAvailableMB()); // should return -1 if per queue setting not set NUnit.Framework.Assert.AreEqual((int)CapacitySchedulerConfiguration.Undefined, csConf .GetMaximumApplicationsPerQueue(queue.GetQueuePath())); int expectedMaxApps = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions * queue.GetAbsoluteCapacity()); NUnit.Framework.Assert.AreEqual(expectedMaxApps, queue.GetMaxApplications()); int expectedMaxAppsPerUser = (int)(expectedMaxApps * (queue.GetUserLimit() / 100.0f ) * queue.GetUserLimitFactor()); NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser ()); // should default to global setting if per queue setting not set NUnit.Framework.Assert.AreEqual((long)CapacitySchedulerConfiguration.DefaultMaximumApplicationmastersResourcePercent , (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent(queue.GetQueuePath ())); // Change the per-queue max AM resources percentage. csConf.SetFloat("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-am-resource-percent" , 0.5f); // Re-create queues to get new configs. queues = new Dictionary <string, CSQueue>(); root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues , TestUtils.spyHook); clusterResource = Resources.CreateResource(100 * 16 * Gb); queue = (LeafQueue)queues[A]; NUnit.Framework.Assert.AreEqual((long)0.5, (long)csConf.GetMaximumApplicationMasterResourcePerQueuePercent (queue.GetQueuePath())); NUnit.Framework.Assert.AreEqual(queue.GetAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(800 * Gb, 1)); NUnit.Framework.Assert.AreEqual(queue.GetUserAMResourceLimit(), Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(400 * Gb, 1)); // Change the per-queue max applications. csConf.SetInt("yarn.scheduler.capacity." + queue.GetQueuePath() + ".maximum-applications" , 9999); // Re-create queues to get new configs. queues = new Dictionary <string, CSQueue>(); root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues , TestUtils.spyHook); queue = (LeafQueue)queues[A]; NUnit.Framework.Assert.AreEqual(9999, (int)csConf.GetMaximumApplicationsPerQueue( queue.GetQueuePath())); NUnit.Framework.Assert.AreEqual(9999, queue.GetMaxApplications()); expectedMaxAppsPerUser = (int)(9999 * (queue.GetUserLimit() / 100.0f) * queue.GetUserLimitFactor ()); NUnit.Framework.Assert.AreEqual(expectedMaxAppsPerUser, queue.GetMaxApplicationsPerUser ()); }