Пример #1
0
        public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, string user, Queue
                                queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
            : base(applicationAttemptId, user, queue, activeUsersManager, rmContext)
        {
            RMApp    rmApp = rmContext.GetRMApps()[GetApplicationId()];
            Resource amResource;

            if (rmApp == null || rmApp.GetAMResourceRequest() == null)
            {
                //the rmApp may be undefined (the resource manager checks for this too)
                //and unmanaged applications do not provide an amResource request
                //in these cases, provide a default using the scheduler
                amResource = rmContext.GetScheduler().GetMinimumResourceCapability();
            }
            else
            {
                amResource = rmApp.GetAMResourceRequest().GetCapability();
            }
            SetAMResource(amResource);
        }
Пример #2
0
        public virtual void TestHeadroom()
        {
            CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();

            csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
            SetupQueueConfiguration(csConf);
            YarnConfiguration        conf      = new YarnConfiguration();
            CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
                                                                           >();

            Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
            Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
            Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
                                                                                          .CreateResource(Gb));
            Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
                                                                                          .CreateResource(16 * Gb));
            Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
                                                                                      .applicationComparator);
            Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
                                                                                .queueComparator);
            Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
                                                                                   );
            Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
            // Say cluster has 100 nodes of 16G each
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
                                                                              (100 * 16 * Gb);
            Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
                                                                                );
            IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();

            CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
                                         .spyHook);
            // Manipulate queue 'a'
            LeafQueue         queue         = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
            string            host_0        = "host_0";
            string            rack_0        = "rack_0";
            FiCaSchedulerNode node_0        = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
            string            user_0        = "user_0";
            string            user_1        = "user_1";
            RecordFactory     recordFactory = RecordFactoryProvider.GetRecordFactory(null);
            RMContext         rmContext     = TestUtils.GetMockRMContext();
            RMContext         spyRMContext  = Org.Mockito.Mockito.Spy(rmContext);
            ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
                                                                                   <ApplicationId, RMApp>());
            RMApp           rmApp             = Org.Mockito.Mockito.Mock <RMApp>();
            ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();

            Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
                                                                         (0, 0);
            Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
                                                                                   );
            Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
                                                                              );
            Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
            Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
            Priority priority_1 = TestUtils.CreateMockPriority(1);
            // Submit first application with some resource-requests from user_0,
            // and check headroom
            ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
                                                                                          0);
            FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
                                                            queue.GetActiveUsersManager(), spyRMContext);

            queue.SubmitApplicationAttempt(app_0_0, user_0);
            IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();

            app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
                                                                     Gb, 2, true, priority_1, recordFactory));
            app_0_0.UpdateResourceRequests(app_0_0_requests);
            // Schedule to compute
            queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                               ));
            Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
                                                                               (10 * 16 * Gb, 1);
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
            // Submit second application from user_0, check headroom
            ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
                                                                                          0);
            FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
                                                            queue.GetActiveUsersManager(), spyRMContext);

            queue.SubmitApplicationAttempt(app_0_1, user_0);
            IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();

            app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
                                                                     Gb, 2, true, priority_1, recordFactory));
            app_0_1.UpdateResourceRequests(app_0_1_requests);
            // Schedule to compute
            queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                               ));
            // Schedule to compute
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
            // no change
            // Submit first application from user_1, check  for new headroom
            ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
                                                                                          0);
            FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
                                                            queue.GetActiveUsersManager(), spyRMContext);

            queue.SubmitApplicationAttempt(app_1_0, user_1);
            IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();

            app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
                                                                     Gb, 2, true, priority_1, recordFactory));
            app_1_0.UpdateResourceRequests(app_1_0_requests);
            // Schedule to compute
            queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                               ));
            // Schedule to compute
            expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
            // changes
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
            // Now reduce cluster size and check for the smaller headroom
            clusterResource = Resources.CreateResource(90 * 16 * Gb);
            queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                               ));
            // Schedule to compute
            expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
            // changes
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
            NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
        }