Ejemplo n.º 1
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="Org.Xml.Sax.SAXException"/>
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
 ///     "/>
 /// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
 public virtual void Initialize(Configuration conf)
 {
     rootQueue = new FSParentQueue("root", scheduler, null);
     queues[rootQueue.GetName()] = rootQueue;
     // Create the default queue
     GetLeafQueue(YarnConfiguration.DefaultQueueName, true);
 }
Ejemplo n.º 2
0
        /// <summary>
        /// This is called after reloading the allocation configuration when the
        /// scheduler is reinitilized
        /// Checks to see whether any non-runnable applications become runnable
        /// now that the max running apps of given queue has been changed
        /// Runs in O(n) where n is the number of apps that are non-runnable and in
        /// the queues that went from having no slack to having slack.
        /// </summary>
        public virtual void UpdateRunnabilityOnReload()
        {
            FSParentQueue rootQueue = scheduler.GetQueueManager().GetRootQueue();
            IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >(
                );

            GatherPossiblyRunnableAppLists(rootQueue, appsNowMaybeRunnable);
            UpdateAppsRunnability(appsNowMaybeRunnable, int.MaxValue);
        }
Ejemplo n.º 3
0
 public FSQueue(string name, FairScheduler scheduler, FSParentQueue parent)
 {
     this.name      = name;
     this.scheduler = scheduler;
     this.metrics   = ((FSQueueMetrics)FSQueueMetrics.ForQueue(GetName(), parent, true,
                                                               scheduler.GetConf()));
     metrics.SetMinShare(GetMinShare());
     metrics.SetMaxShare(GetMaxShare());
     this.parent = parent;
 }
Ejemplo n.º 4
0
        /// <summary>
        /// Checks to see whether any other applications runnable now that the given
        /// application has been removed from the given queue.
        /// </summary>
        /// <remarks>
        /// Checks to see whether any other applications runnable now that the given
        /// application has been removed from the given queue.  And makes them so.
        /// Runs in O(n log(n)) where n is the number of queues that are under the
        /// highest queue that went from having no slack to having slack.
        /// </remarks>
        public virtual void UpdateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue
                                                          )
        {
            AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration();
            // childqueueX might have no pending apps itself, but if a queue higher up
            // in the hierarchy parentqueueY has a maxRunningApps set, an app completion
            // in childqueueX could allow an app in some other distant child of
            // parentqueueY to become runnable.
            // An app removal will only possibly allow another app to become runnable if
            // the queue was already at its max before the removal.
            // Thus we find the ancestor queue highest in the tree for which the app
            // that was at its maxRunningApps before the removal.
            FSQueue highestQueueWithAppsNowRunnable = (queue.GetNumRunnableApps() == allocConf
                                                       .GetQueueMaxApps(queue.GetName()) - 1) ? queue : null;
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                if (parent.GetNumRunnableApps() == allocConf.GetQueueMaxApps(parent.GetName()) -
                    1)
                {
                    highestQueueWithAppsNowRunnable = parent;
                }
                parent = parent.GetParent();
            }
            IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >(
                );

            // Compile lists of apps which may now be runnable
            // We gather lists instead of building a set of all non-runnable apps so
            // that this whole operation can be O(number of queues) instead of
            // O(number of apps)
            if (highestQueueWithAppsNowRunnable != null)
            {
                GatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable
                                               );
            }
            string user           = app.GetUser();
            int    userNumRunning = usersNumRunnableApps[user];

            if (userNumRunning == null)
            {
                userNumRunning = 0;
            }
            if (userNumRunning == allocConf.GetUserMaxApps(user) - 1)
            {
                IList <FSAppAttempt> userWaitingApps = usersNonRunnableApps.Get(user);
                if (userWaitingApps != null)
                {
                    appsNowMaybeRunnable.AddItem(userWaitingApps);
                }
            }
            UpdateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.Count);
        }
Ejemplo n.º 5
0
 public FSLeafQueue(string name, FairScheduler scheduler, FSParentQueue parent)
     : base(name, scheduler, parent)
 {
     // apps that are runnable
     // get a lock with fair distribution for app list updates
     // Variables used for preemption
     // Track the AM resource usage for this queue
     this.lastTimeAtMinShare           = scheduler.GetClock().GetTime();
     this.lastTimeAtFairShareThreshold = scheduler.GetClock().GetTime();
     activeUsersManager = new ActiveUsersManager(GetMetrics());
     amResourceUsage    = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(0, 0);
 }
Ejemplo n.º 6
0
        /// <summary>
        /// Tracks the given new runnable app for purposes of maintaining max running
        /// app limits.
        /// </summary>
        public virtual void TrackRunnableApp(FSAppAttempt app)
        {
            string      user  = app.GetUser();
            FSLeafQueue queue = ((FSLeafQueue)app.GetQueue());
            // Increment running counts for all parent queues
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                parent.IncrementRunnableApps();
                parent = parent.GetParent();
            }
            int userNumRunnable = usersNumRunnableApps[user];

            usersNumRunnableApps[user] = (userNumRunnable == null ? 0 : userNumRunnable) + 1;
        }
Ejemplo n.º 7
0
        /// <summary>
        /// Updates the relevant tracking variables after a runnable app with the given
        /// queue and user has been removed.
        /// </summary>
        public virtual void UntrackRunnableApp(FSAppAttempt app)
        {
            // Update usersRunnableApps
            string user = app.GetUser();
            int    newUserNumRunning = usersNumRunnableApps[user] - 1;

            if (newUserNumRunning == 0)
            {
                Sharpen.Collections.Remove(usersNumRunnableApps, user);
            }
            else
            {
                usersNumRunnableApps[user] = newUserNumRunning;
            }
            // Update runnable app bookkeeping for queues
            FSLeafQueue   queue  = ((FSLeafQueue)app.GetQueue());
            FSParentQueue parent = queue.GetParent();

            while (parent != null)
            {
                parent.DecrementRunnableApps();
                parent = parent.GetParent();
            }
        }
Ejemplo n.º 8
0
        /// <summary>
        /// Creates a leaf or parent queue based on what is specified in 'queueType'
        /// and places it in the tree.
        /// </summary>
        /// <remarks>
        /// Creates a leaf or parent queue based on what is specified in 'queueType'
        /// and places it in the tree. Creates any parents that don't already exist.
        /// </remarks>
        /// <returns>
        /// the created queue, if successful. null if not allowed (one of the parent
        /// queues in the queue name is already a leaf queue)
        /// </returns>
        private FSQueue CreateQueue(string name, FSQueueType queueType)
        {
            IList <string> newQueueNames = new AList <string>();

            newQueueNames.AddItem(name);
            int           sepIndex = name.Length;
            FSParentQueue parent   = null;

            // Move up the queue tree until we reach one that exists.
            while (sepIndex != -1)
            {
                sepIndex = name.LastIndexOf('.', sepIndex - 1);
                FSQueue queue;
                string  curName = null;
                curName = Sharpen.Runtime.Substring(name, 0, sepIndex);
                queue   = queues[curName];
                if (queue == null)
                {
                    newQueueNames.AddItem(curName);
                }
                else
                {
                    if (queue is FSParentQueue)
                    {
                        parent = (FSParentQueue)queue;
                        break;
                    }
                    else
                    {
                        return(null);
                    }
                }
            }
            // At this point, parent refers to the deepest existing parent of the
            // queue to create.
            // Now that we know everything worked out, make all the queues
            // and add them to the map.
            AllocationConfiguration queueConf = scheduler.GetAllocationConfiguration();
            FSLeafQueue             leafQueue = null;

            for (int i = newQueueNames.Count - 1; i >= 0; i--)
            {
                string queueName = newQueueNames[i];
                if (i == 0 && queueType != FSQueueType.Parent)
                {
                    leafQueue = new FSLeafQueue(name, scheduler, parent);
                    try
                    {
                        leafQueue.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
                    }
                    catch (AllocationConfigurationException ex)
                    {
                        Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
                                     () + " on new leaf queue.", ex);
                    }
                    parent.AddChildQueue(leafQueue);
                    queues[leafQueue.GetName()] = leafQueue;
                    leafQueues.AddItem(leafQueue);
                    leafQueue.UpdatePreemptionVariables();
                    return(leafQueue);
                }
                else
                {
                    FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent);
                    try
                    {
                        newParent.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
                    }
                    catch (AllocationConfigurationException ex)
                    {
                        Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
                                     () + " on new parent queue.", ex);
                    }
                    parent.AddChildQueue(newParent);
                    queues[newParent.GetName()] = newParent;
                    newParent.UpdatePreemptionVariables();
                    parent = newParent;
                }
            }
            return(parent);
        }