public virtual void TestHeadroom()
        {
            FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>();

            Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock(
                                                                              ));
            FSLeafQueue mockQueue         = Org.Mockito.Mockito.Mock <FSLeafQueue>();
            Resource    queueMaxResources = Resource.NewInstance(5 * 1024, 3);

            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource
                                                                             (4096, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                     .NewInstance(2048, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract(
                queueFairShare, queueUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources
                                                                                     .Subtract(queueMaxResources, queueUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
                                                                              (8192, 8);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource
                                                                           (2048, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract
                                                                               (clusterResource, clusterUsage);
            QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();

            Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources);
            Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare);
            Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage);
            Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource
                                                                                    );
            Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn
                (clusterUsage);
            Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics
                                                                                     );
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            FSAppAttempt         schedulerApp         = new FSAppAttempt(mockScheduler, applicationAttemptId,
                                                                         "user1", mockQueue, null, rmContext);

            // Min of Memory and CPU across cluster and queue is used in
            // DominantResourceFairnessPolicy
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(DominantResourceFairnessPolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores
                                                                                                      (), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores
                                                                                                      ()));
            // Fair and Fifo ignore CPU of queue, so use cluster available CPU
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(FairSharePolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
                                                                                                           (), queueMaxResourcesAvailable.GetVirtualCores()));
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(FifoPolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
                                                                                                           (), queueMaxResourcesAvailable.GetVirtualCores()));
        }
Exemple #2
0
 public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId
                     , string user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext
                     rmContext)
     : base(applicationAttemptId, user, queue, activeUsersManager, rmContext)
 {
     this.scheduler       = scheduler;
     this.startTime       = scheduler.GetClock().GetTime();
     this.priority        = Priority.NewInstance(1);
     this.resourceWeights = new ResourceWeights();
 }
        /// <summary>
        /// Checks to see whether any other applications runnable now that the given
        /// application has been removed from the given queue.
        /// </summary>
        /// <remarks>
        /// Checks to see whether any other applications runnable now that the given
        /// application has been removed from the given queue.  And makes them so.
        /// Runs in O(n log(n)) where n is the number of queues that are under the
        /// highest queue that went from having no slack to having slack.
        /// </remarks>
        public virtual void UpdateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue
                                                          )
        {
            AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration();
            // childqueueX might have no pending apps itself, but if a queue higher up
            // in the hierarchy parentqueueY has a maxRunningApps set, an app completion
            // in childqueueX could allow an app in some other distant child of
            // parentqueueY to become runnable.
            // An app removal will only possibly allow another app to become runnable if
            // the queue was already at its max before the removal.
            // Thus we find the ancestor queue highest in the tree for which the app
            // that was at its maxRunningApps before the removal.
            FSQueue highestQueueWithAppsNowRunnable = (queue.GetNumRunnableApps() == allocConf
                                                       .GetQueueMaxApps(queue.GetName()) - 1) ? queue : null;
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                if (parent.GetNumRunnableApps() == allocConf.GetQueueMaxApps(parent.GetName()) -
                    1)
                {
                    highestQueueWithAppsNowRunnable = parent;
                }
                parent = parent.GetParent();
            }
            IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >(
                );

            // Compile lists of apps which may now be runnable
            // We gather lists instead of building a set of all non-runnable apps so
            // that this whole operation can be O(number of queues) instead of
            // O(number of apps)
            if (highestQueueWithAppsNowRunnable != null)
            {
                GatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable
                                               );
            }
            string user           = app.GetUser();
            int    userNumRunning = usersNumRunnableApps[user];

            if (userNumRunning == null)
            {
                userNumRunning = 0;
            }
            if (userNumRunning == allocConf.GetUserMaxApps(user) - 1)
            {
                IList <FSAppAttempt> userWaitingApps = usersNonRunnableApps.Get(user);
                if (userWaitingApps != null)
                {
                    appsNowMaybeRunnable.AddItem(userWaitingApps);
                }
            }
            UpdateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.Count);
        }
Exemple #4
0
        public virtual void TestConcurrentAccess()
        {
            conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
            resourceManager = new MockRM(conf);
            resourceManager.Start();
            scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
            string      queueName   = "root.queue1";
            FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true
                                                                               );
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            FSAppAttempt         app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable
                                                        , null, rmContext);
            // this needs to be in sync with the number of runnables declared below
            int testThreads            = 2;
            IList <Runnable> runnables = new AList <Runnable>();

            // add applications to modify the list
            runnables.AddItem(new _Runnable_257(schedulable, app));
            // iterate over the list a couple of times in a different thread
            runnables.AddItem(new _Runnable_267(schedulable));
            IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception
                                                                                           >());
            ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads);

            try
            {
                CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads);
                CountDownLatch startBlocker            = new CountDownLatch(1);
                CountDownLatch allDone = new CountDownLatch(testThreads);
                foreach (Runnable submittedTestRunnable in runnables)
                {
                    threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable
                                                        , exceptions, allDone));
                }
                // wait until all threads are ready
                allExecutorThreadsReady.Await();
                // start all test runners
                startBlocker.CountDown();
                int testTimeout = 2;
                NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds"
                                              , allDone.Await(testTimeout, TimeUnit.Seconds));
            }
            catch (Exception ie)
            {
                exceptions.AddItem(ie);
            }
            finally
            {
                threadPool.ShutdownNow();
            }
            NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions
                                          .IsEmpty());
        }
        public virtual void TestDelayScheduling()
        {
            FSLeafQueue queue = Org.Mockito.Mockito.Mock <FSLeafQueue>();
            Priority    prio  = Org.Mockito.Mockito.Mock <Priority>();

            Org.Mockito.Mockito.When(prio.GetPriority()).ThenReturn(1);
            double nodeLocalityThreshold = .5;
            double rackLocalityThreshold = .6;
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            FSAppAttempt         schedulerApp         = new FSAppAttempt(scheduler, applicationAttemptId, "user1"
                                                                         , queue, null, rmContext);

            // Default level should be node-local
            NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            // First five scheduling opportunities should remain node local
            for (int i = 0; i < 5; i++)
            {
                schedulerApp.AddSchedulingOpportunity(prio);
                NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevel
                                                    (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            }
            // After five it should switch to rack local
            schedulerApp.AddSchedulingOpportunity(prio);
            NUnit.Framework.Assert.AreEqual(NodeType.RackLocal, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            // Manually set back to node local
            schedulerApp.ResetAllowedLocalityLevel(prio, NodeType.NodeLocal);
            schedulerApp.ResetSchedulingOpportunities(prio);
            NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            // Now escalate again to rack-local, then to off-switch
            for (int i_1 = 0; i_1 < 5; i_1++)
            {
                schedulerApp.AddSchedulingOpportunity(prio);
                NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevel
                                                    (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            }
            schedulerApp.AddSchedulingOpportunity(prio);
            NUnit.Framework.Assert.AreEqual(NodeType.RackLocal, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            for (int i_2 = 0; i_2 < 6; i_2++)
            {
                schedulerApp.AddSchedulingOpportunity(prio);
                NUnit.Framework.Assert.AreEqual(NodeType.RackLocal, schedulerApp.GetAllowedLocalityLevel
                                                    (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
            }
            schedulerApp.AddSchedulingOpportunity(prio);
            NUnit.Framework.Assert.AreEqual(NodeType.OffSwitch, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
        }
        public virtual void TestLocalityLevelWithoutDelays()
        {
            FSLeafQueue queue = Org.Mockito.Mockito.Mock <FSLeafQueue>();
            Priority    prio  = Org.Mockito.Mockito.Mock <Priority>();

            Org.Mockito.Mockito.When(prio.GetPriority()).ThenReturn(1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            FSAppAttempt         schedulerApp         = new FSAppAttempt(scheduler, applicationAttemptId, "user1"
                                                                         , queue, null, rmContext);

            NUnit.Framework.Assert.AreEqual(NodeType.OffSwitch, schedulerApp.GetAllowedLocalityLevel
                                                (prio, 10, -1.0, -1.0));
        }
Exemple #7
0
        /// <exception cref="System.Exception"/>
        public virtual void Test()
        {
            conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
            PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));

            @out.WriteLine("<?xml version=\"1.0\"?>");
            @out.WriteLine("<allocations>");
            @out.WriteLine("<queue name=\"queueA\">");
            @out.WriteLine("<minResources>2048mb,0vcores</minResources>");
            @out.WriteLine("</queue>");
            @out.WriteLine("<queue name=\"queueB\">");
            @out.WriteLine("<minResources>2048mb,0vcores</minResources>");
            @out.WriteLine("</queue>");
            @out.WriteLine("</allocations>");
            @out.Close();
            resourceManager = new MockRM(conf);
            resourceManager.Start();
            scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
            // Add one big node (only care about aggregate capacity)
            RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4 * 1024, 4), 1,
                                                 "127.0.0.1");
            NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);

            scheduler.Handle(nodeEvent1);
            scheduler.Update();
            // Queue A wants 3 * 1024. Node update gives this all to A
            CreateSchedulingRequest(3 * 1024, "queueA", "user1");
            scheduler.Update();
            NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);

            scheduler.Handle(nodeEvent2);
            // Queue B arrives and wants 1 * 1024
            CreateSchedulingRequest(1 * 1024, "queueB", "user1");
            scheduler.Update();
            ICollection <FSLeafQueue> queues = scheduler.GetQueueManager().GetLeafQueues();

            NUnit.Framework.Assert.AreEqual(3, queues.Count);
            // Queue A should be above min share, B below.
            FSLeafQueue queueA = scheduler.GetQueueManager().GetLeafQueue("queueA", false);
            FSLeafQueue queueB = scheduler.GetQueueManager().GetLeafQueue("queueB", false);

            NUnit.Framework.Assert.IsFalse(queueA.IsStarvedForMinShare());
            NUnit.Framework.Assert.IsTrue(queueB.IsStarvedForMinShare());
            // Node checks in again, should allocate for B
            scheduler.Handle(nodeEvent2);
            // Now B should have min share ( = demand here)
            NUnit.Framework.Assert.IsFalse(queueB.IsStarvedForMinShare());
        }
        /// <summary>
        /// Tracks the given new runnable app for purposes of maintaining max running
        /// app limits.
        /// </summary>
        public virtual void TrackRunnableApp(FSAppAttempt app)
        {
            string      user  = app.GetUser();
            FSLeafQueue queue = ((FSLeafQueue)app.GetQueue());
            // Increment running counts for all parent queues
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                parent.IncrementRunnableApps();
                parent = parent.GetParent();
            }
            int userNumRunnable = usersNumRunnableApps[user];

            usersNumRunnableApps[user] = (userNumRunnable == null ? 0 : userNumRunnable) + 1;
        }
Exemple #9
0
 /// <summary>
 /// Returns true if there are no applications, running or not, in the given
 /// queue or any of its descendents.
 /// </summary>
 protected internal virtual bool IsEmpty(FSQueue queue)
 {
     if (queue is FSLeafQueue)
     {
         FSLeafQueue leafQueue = (FSLeafQueue)queue;
         return(queue.GetNumRunnableApps() == 0 && leafQueue.GetNumNonRunnableApps() == 0);
     }
     else
     {
         foreach (FSQueue child in queue.GetChildQueues())
         {
             if (!IsEmpty(child))
             {
                 return(false);
             }
         }
         return(true);
     }
 }
        private FSAppAttempt AddApp(FSLeafQueue queue, string user)
        {
            ApplicationId        appId = ApplicationId.NewInstance(0l, appNum++);
            ApplicationAttemptId attId = ApplicationAttemptId.NewInstance(appId, 0);
            bool         runnable      = maxAppsEnforcer.CanAppBeRunnable(queue, user);
            FSAppAttempt app           = new FSAppAttempt(scheduler, attId, user, queue, null, rmContext
                                                          );

            queue.AddApp(app, runnable);
            if (runnable)
            {
                maxAppsEnforcer.TrackRunnableApp(app);
            }
            else
            {
                maxAppsEnforcer.TrackNonRunnableApp(app);
            }
            return(app);
        }
        public virtual void TestRemoveEnablesAppOnCousinQueue()
        {
            FSLeafQueue leaf1 = queueManager.GetLeafQueue("root.queue1.subqueue1.leaf1", true
                                                          );
            FSLeafQueue leaf2 = queueManager.GetLeafQueue("root.queue1.subqueue2.leaf2", true
                                                          );

            queueMaxApps["root.queue1"] = 2;
            FSAppAttempt app1 = AddApp(leaf1, "user");

            AddApp(leaf2, "user");
            AddApp(leaf2, "user");
            NUnit.Framework.Assert.AreEqual(1, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumNonRunnableApps());
            RemoveApp(app1);
            NUnit.Framework.Assert.AreEqual(0, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(2, leaf2.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(0, leaf2.GetNumNonRunnableApps());
        }
        public virtual void TestRemoveDoesNotEnableAnyApp()
        {
            FSLeafQueue leaf1 = queueManager.GetLeafQueue("root.queue1", true);
            FSLeafQueue leaf2 = queueManager.GetLeafQueue("root.queue2", true);

            queueMaxApps["root"]        = 2;
            queueMaxApps["root.queue1"] = 1;
            queueMaxApps["root.queue2"] = 1;
            FSAppAttempt app1 = AddApp(leaf1, "user");

            AddApp(leaf2, "user");
            AddApp(leaf2, "user");
            NUnit.Framework.Assert.AreEqual(1, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumNonRunnableApps());
            RemoveApp(app1);
            NUnit.Framework.Assert.AreEqual(0, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumNonRunnableApps());
        }
        public virtual void TestDelaySchedulingForContinuousScheduling()
        {
            FSLeafQueue queue = scheduler.GetQueueManager().GetLeafQueue("queue", true);
            Priority    prio  = Org.Mockito.Mockito.Mock <Priority>();

            Org.Mockito.Mockito.When(prio.GetPriority()).ThenReturn(1);
            TestFSAppAttempt.MockClock clock = new TestFSAppAttempt.MockClock(this);
            scheduler.SetClock(clock);
            long nodeLocalityDelayMs = 5 * 1000L;
            // 5 seconds
            long rackLocalityDelayMs = 6 * 1000L;
            // 6 seconds
            RMContext            rmContext            = resourceManager.GetRMContext();
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            FSAppAttempt         schedulerApp         = new FSAppAttempt(scheduler, applicationAttemptId, "user1"
                                                                         , queue, null, rmContext);

            // Default level should be node-local
            NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
            // after 4 seconds should remain node local
            clock.Tick(4);
            NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
            // after 6 seconds should switch to rack local
            clock.Tick(2);
            NUnit.Framework.Assert.AreEqual(NodeType.RackLocal, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
            // manually set back to node local
            schedulerApp.ResetAllowedLocalityLevel(prio, NodeType.NodeLocal);
            schedulerApp.ResetSchedulingOpportunities(prio, clock.GetTime());
            NUnit.Framework.Assert.AreEqual(NodeType.NodeLocal, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
            // Now escalate again to rack-local, then to off-switch
            clock.Tick(6);
            NUnit.Framework.Assert.AreEqual(NodeType.RackLocal, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
            clock.Tick(7);
            NUnit.Framework.Assert.AreEqual(NodeType.OffSwitch, schedulerApp.GetAllowedLocalityLevelByTime
                                                (prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.GetTime()));
        }
        public virtual void TestRemoveEnablesOneByQueueOneByUser()
        {
            FSLeafQueue leaf1 = queueManager.GetLeafQueue("root.queue1.leaf1", true);
            FSLeafQueue leaf2 = queueManager.GetLeafQueue("root.queue1.leaf2", true);

            queueMaxApps["root.queue1.leaf1"] = 2;
            userMaxApps["user1"] = 1;
            FSAppAttempt app1 = AddApp(leaf1, "user1");

            AddApp(leaf1, "user2");
            AddApp(leaf1, "user3");
            AddApp(leaf2, "user1");
            NUnit.Framework.Assert.AreEqual(2, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf1.GetNumNonRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumNonRunnableApps());
            RemoveApp(app1);
            NUnit.Framework.Assert.AreEqual(2, leaf1.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(1, leaf2.GetNumRunnableApps());
            NUnit.Framework.Assert.AreEqual(0, leaf1.GetNumNonRunnableApps());
            NUnit.Framework.Assert.AreEqual(0, leaf2.GetNumNonRunnableApps());
        }
Exemple #15
0
        public virtual void TestUpdateDemand()
        {
            conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
            resourceManager = new MockRM(conf);
            resourceManager.Start();
            scheduler           = (FairScheduler)resourceManager.GetResourceScheduler();
            scheduler.allocConf = Org.Mockito.Mockito.Mock <AllocationConfiguration>();
            string queueName = "root.queue1";

            Org.Mockito.Mockito.When(scheduler.allocConf.GetMaxResources(queueName)).ThenReturn
                (maxResource);
            Org.Mockito.Mockito.When(scheduler.allocConf.GetMinResources(queueName)).ThenReturn
                (Resources.None());
            FSLeafQueue  schedulable = new FSLeafQueue(queueName, scheduler, null);
            FSAppAttempt app         = Org.Mockito.Mockito.Mock <FSAppAttempt>();

            Org.Mockito.Mockito.When(app.GetDemand()).ThenReturn(maxResource);
            schedulable.AddAppSchedulable(app);
            schedulable.AddAppSchedulable(app);
            schedulable.UpdateDemand();
            NUnit.Framework.Assert.IsTrue("Demand is greater than max allowed ", Resources.Equals
                                              (schedulable.GetDemand(), maxResource));
        }
        /// <summary>
        /// Updates the relevant tracking variables after a runnable app with the given
        /// queue and user has been removed.
        /// </summary>
        public virtual void UntrackRunnableApp(FSAppAttempt app)
        {
            // Update usersRunnableApps
            string user = app.GetUser();
            int    newUserNumRunning = usersNumRunnableApps[user] - 1;

            if (newUserNumRunning == 0)
            {
                Sharpen.Collections.Remove(usersNumRunnableApps, user);
            }
            else
            {
                usersNumRunnableApps[user] = newUserNumRunning;
            }
            // Update runnable app bookkeeping for queues
            FSLeafQueue   queue  = ((FSLeafQueue)app.GetQueue());
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                parent.DecrementRunnableApps();
                parent = parent.GetParent();
            }
        }
Exemple #17
0
 public _Runnable_257(FSLeafQueue schedulable, FSAppAttempt app)
 {
     this.schedulable = schedulable;
     this.app         = app;
 }
Exemple #18
0
 public _Runnable_267(FSLeafQueue schedulable)
 {
     this.schedulable = schedulable;
 }
Exemple #19
0
        /// <summary>
        /// Creates a leaf or parent queue based on what is specified in 'queueType'
        /// and places it in the tree.
        /// </summary>
        /// <remarks>
        /// Creates a leaf or parent queue based on what is specified in 'queueType'
        /// and places it in the tree. Creates any parents that don't already exist.
        /// </remarks>
        /// <returns>
        /// the created queue, if successful. null if not allowed (one of the parent
        /// queues in the queue name is already a leaf queue)
        /// </returns>
        private FSQueue CreateQueue(string name, FSQueueType queueType)
        {
            IList <string> newQueueNames = new AList <string>();

            newQueueNames.AddItem(name);
            int           sepIndex = name.Length;
            FSParentQueue parent   = null;

            // Move up the queue tree until we reach one that exists.
            while (sepIndex != -1)
            {
                sepIndex = name.LastIndexOf('.', sepIndex - 1);
                FSQueue queue;
                string  curName = null;
                curName = Sharpen.Runtime.Substring(name, 0, sepIndex);
                queue   = queues[curName];
                if (queue == null)
                {
                    newQueueNames.AddItem(curName);
                }
                else
                {
                    if (queue is FSParentQueue)
                    {
                        parent = (FSParentQueue)queue;
                        break;
                    }
                    else
                    {
                        return(null);
                    }
                }
            }
            // At this point, parent refers to the deepest existing parent of the
            // queue to create.
            // Now that we know everything worked out, make all the queues
            // and add them to the map.
            AllocationConfiguration queueConf = scheduler.GetAllocationConfiguration();
            FSLeafQueue             leafQueue = null;

            for (int i = newQueueNames.Count - 1; i >= 0; i--)
            {
                string queueName = newQueueNames[i];
                if (i == 0 && queueType != FSQueueType.Parent)
                {
                    leafQueue = new FSLeafQueue(name, scheduler, parent);
                    try
                    {
                        leafQueue.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
                    }
                    catch (AllocationConfigurationException ex)
                    {
                        Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
                                     () + " on new leaf queue.", ex);
                    }
                    parent.AddChildQueue(leafQueue);
                    queues[leafQueue.GetName()] = leafQueue;
                    leafQueues.AddItem(leafQueue);
                    leafQueue.UpdatePreemptionVariables();
                    return(leafQueue);
                }
                else
                {
                    FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent);
                    try
                    {
                        newParent.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
                    }
                    catch (AllocationConfigurationException ex)
                    {
                        Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
                                     () + " on new parent queue.", ex);
                    }
                    parent.AddChildQueue(newParent);
                    queues[newParent.GetName()] = newParent;
                    newParent.UpdatePreemptionVariables();
                    parent = newParent;
                }
            }
            return(parent);
        }
Exemple #20
0
        /// <exception cref="System.Exception"/>
        public virtual void TestIsStarvedForFairShare()
        {
            conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
            PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));

            @out.WriteLine("<?xml version=\"1.0\"?>");
            @out.WriteLine("<allocations>");
            @out.WriteLine("<queue name=\"queueA\">");
            @out.WriteLine("<weight>.2</weight>");
            @out.WriteLine("</queue>");
            @out.WriteLine("<queue name=\"queueB\">");
            @out.WriteLine("<weight>.8</weight>");
            @out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>");
            @out.WriteLine("<queue name=\"queueB1\">");
            @out.WriteLine("</queue>");
            @out.WriteLine("<queue name=\"queueB2\">");
            @out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>");
            @out.WriteLine("</queue>");
            @out.WriteLine("</queue>");
            @out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>"
                           );
            @out.WriteLine("</allocations>");
            @out.Close();
            resourceManager = new MockRM(conf);
            resourceManager.Start();
            scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
            // Add one big node (only care about aggregate capacity)
            RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10),
                                                 1, "127.0.0.1");
            NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);

            scheduler.Handle(nodeEvent1);
            scheduler.Update();
            // Queue A wants 4 * 1024. Node update gives this all to A
            CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4);
            scheduler.Update();
            NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);

            for (int i = 0; i < 4; i++)
            {
                scheduler.Handle(nodeEvent2);
            }
            QueueManager queueMgr = scheduler.GetQueueManager();
            FSLeafQueue  queueA   = queueMgr.GetLeafQueue("queueA", false);

            NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory());
            // Both queue B1 and queue B2 want 3 * 1024
            CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3);
            CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3);
            scheduler.Update();
            for (int i_1 = 0; i_1 < 4; i_1++)
            {
                scheduler.Handle(nodeEvent2);
            }
            FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false);
            FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false);

            NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory());
            NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory());
            // For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share
            // threshold is 1.6 * 1024
            NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
            // For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share
            // threshold is 2.4 * 1024
            NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare());
            // Node checks in again
            scheduler.Handle(nodeEvent2);
            scheduler.Handle(nodeEvent2);
            NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory());
            NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory());
            // Both queue B1 and queue B2 usages go to 3 * 1024
            NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
            NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare());
        }