/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Xml.Sax.SAXException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException /// "/> /// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/> public virtual void Initialize(Configuration conf) { rootQueue = new FSParentQueue("root", scheduler, null); queues[rootQueue.GetName()] = rootQueue; // Create the default queue GetLeafQueue(YarnConfiguration.DefaultQueueName, true); }
/// <summary> /// This is called after reloading the allocation configuration when the /// scheduler is reinitilized /// Checks to see whether any non-runnable applications become runnable /// now that the max running apps of given queue has been changed /// Runs in O(n) where n is the number of apps that are non-runnable and in /// the queues that went from having no slack to having slack. /// </summary> public virtual void UpdateRunnabilityOnReload() { FSParentQueue rootQueue = scheduler.GetQueueManager().GetRootQueue(); IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >( ); GatherPossiblyRunnableAppLists(rootQueue, appsNowMaybeRunnable); UpdateAppsRunnability(appsNowMaybeRunnable, int.MaxValue); }
public FSQueue(string name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; this.metrics = ((FSQueueMetrics)FSQueueMetrics.ForQueue(GetName(), parent, true, scheduler.GetConf())); metrics.SetMinShare(GetMinShare()); metrics.SetMaxShare(GetMaxShare()); this.parent = parent; }
/// <summary> /// Checks to see whether any other applications runnable now that the given /// application has been removed from the given queue. /// </summary> /// <remarks> /// Checks to see whether any other applications runnable now that the given /// application has been removed from the given queue. And makes them so. /// Runs in O(n log(n)) where n is the number of queues that are under the /// highest queue that went from having no slack to having slack. /// </remarks> public virtual void UpdateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue ) { AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration(); // childqueueX might have no pending apps itself, but if a queue higher up // in the hierarchy parentqueueY has a maxRunningApps set, an app completion // in childqueueX could allow an app in some other distant child of // parentqueueY to become runnable. // An app removal will only possibly allow another app to become runnable if // the queue was already at its max before the removal. // Thus we find the ancestor queue highest in the tree for which the app // that was at its maxRunningApps before the removal. FSQueue highestQueueWithAppsNowRunnable = (queue.GetNumRunnableApps() == allocConf .GetQueueMaxApps(queue.GetName()) - 1) ? queue : null; FSParentQueue parent = queue.GetParent(); while (parent != null) { if (parent.GetNumRunnableApps() == allocConf.GetQueueMaxApps(parent.GetName()) - 1) { highestQueueWithAppsNowRunnable = parent; } parent = parent.GetParent(); } IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >( ); // Compile lists of apps which may now be runnable // We gather lists instead of building a set of all non-runnable apps so // that this whole operation can be O(number of queues) instead of // O(number of apps) if (highestQueueWithAppsNowRunnable != null) { GatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable ); } string user = app.GetUser(); int userNumRunning = usersNumRunnableApps[user]; if (userNumRunning == null) { userNumRunning = 0; } if (userNumRunning == allocConf.GetUserMaxApps(user) - 1) { IList <FSAppAttempt> userWaitingApps = usersNonRunnableApps.Get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.AddItem(userWaitingApps); } } UpdateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.Count); }
public FSLeafQueue(string name, FairScheduler scheduler, FSParentQueue parent) : base(name, scheduler, parent) { // apps that are runnable // get a lock with fair distribution for app list updates // Variables used for preemption // Track the AM resource usage for this queue this.lastTimeAtMinShare = scheduler.GetClock().GetTime(); this.lastTimeAtFairShareThreshold = scheduler.GetClock().GetTime(); activeUsersManager = new ActiveUsersManager(GetMetrics()); amResourceUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(0, 0); }
/// <summary> /// Tracks the given new runnable app for purposes of maintaining max running /// app limits. /// </summary> public virtual void TrackRunnableApp(FSAppAttempt app) { string user = app.GetUser(); FSLeafQueue queue = ((FSLeafQueue)app.GetQueue()); // Increment running counts for all parent queues FSParentQueue parent = queue.GetParent(); while (parent != null) { parent.IncrementRunnableApps(); parent = parent.GetParent(); } int userNumRunnable = usersNumRunnableApps[user]; usersNumRunnableApps[user] = (userNumRunnable == null ? 0 : userNumRunnable) + 1; }
/// <summary> /// Updates the relevant tracking variables after a runnable app with the given /// queue and user has been removed. /// </summary> public virtual void UntrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps string user = app.GetUser(); int newUserNumRunning = usersNumRunnableApps[user] - 1; if (newUserNumRunning == 0) { Sharpen.Collections.Remove(usersNumRunnableApps, user); } else { usersNumRunnableApps[user] = newUserNumRunning; } // Update runnable app bookkeeping for queues FSLeafQueue queue = ((FSLeafQueue)app.GetQueue()); FSParentQueue parent = queue.GetParent(); while (parent != null) { parent.DecrementRunnableApps(); parent = parent.GetParent(); } }
/// <summary> /// Creates a leaf or parent queue based on what is specified in 'queueType' /// and places it in the tree. /// </summary> /// <remarks> /// Creates a leaf or parent queue based on what is specified in 'queueType' /// and places it in the tree. Creates any parents that don't already exist. /// </remarks> /// <returns> /// the created queue, if successful. null if not allowed (one of the parent /// queues in the queue name is already a leaf queue) /// </returns> private FSQueue CreateQueue(string name, FSQueueType queueType) { IList <string> newQueueNames = new AList <string>(); newQueueNames.AddItem(name); int sepIndex = name.Length; FSParentQueue parent = null; // Move up the queue tree until we reach one that exists. while (sepIndex != -1) { sepIndex = name.LastIndexOf('.', sepIndex - 1); FSQueue queue; string curName = null; curName = Sharpen.Runtime.Substring(name, 0, sepIndex); queue = queues[curName]; if (queue == null) { newQueueNames.AddItem(curName); } else { if (queue is FSParentQueue) { parent = (FSParentQueue)queue; break; } else { return(null); } } } // At this point, parent refers to the deepest existing parent of the // queue to create. // Now that we know everything worked out, make all the queues // and add them to the map. AllocationConfiguration queueConf = scheduler.GetAllocationConfiguration(); FSLeafQueue leafQueue = null; for (int i = newQueueNames.Count - 1; i >= 0; i--) { string queueName = newQueueNames[i]; if (i == 0 && queueType != FSQueueType.Parent) { leafQueue = new FSLeafQueue(name, scheduler, parent); try { leafQueue.SetPolicy(queueConf.GetDefaultSchedulingPolicy()); } catch (AllocationConfigurationException ex) { Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy () + " on new leaf queue.", ex); } parent.AddChildQueue(leafQueue); queues[leafQueue.GetName()] = leafQueue; leafQueues.AddItem(leafQueue); leafQueue.UpdatePreemptionVariables(); return(leafQueue); } else { FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent); try { newParent.SetPolicy(queueConf.GetDefaultSchedulingPolicy()); } catch (AllocationConfigurationException ex) { Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy () + " on new parent queue.", ex); } parent.AddChildQueue(newParent); queues[newParent.GetName()] = newParent; newParent.UpdatePreemptionVariables(); parent = newParent; } } return(parent); }