/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Xml.Sax.SAXException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
public virtual void Initialize(Configuration conf)
{
rootQueue = new FSParentQueue("root", scheduler, null);
queues[rootQueue.GetName()] = rootQueue;
// Create the default queue
GetLeafQueue(YarnConfiguration.DefaultQueueName, true);
}
/// <summary>
/// This is called after reloading the allocation configuration when the
/// scheduler is reinitilized
/// Checks to see whether any non-runnable applications become runnable
/// now that the max running apps of given queue has been changed
/// Runs in O(n) where n is the number of apps that are non-runnable and in
/// the queues that went from having no slack to having slack.
/// </summary>
public virtual void UpdateRunnabilityOnReload()
{
FSParentQueue rootQueue = scheduler.GetQueueManager().GetRootQueue();
IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >(
);
GatherPossiblyRunnableAppLists(rootQueue, appsNowMaybeRunnable);
UpdateAppsRunnability(appsNowMaybeRunnable, int.MaxValue);
}
public FSQueue(string name, FairScheduler scheduler, FSParentQueue parent)
{
this.name = name;
this.scheduler = scheduler;
this.metrics = ((FSQueueMetrics)FSQueueMetrics.ForQueue(GetName(), parent, true,
scheduler.GetConf()));
metrics.SetMinShare(GetMinShare());
metrics.SetMaxShare(GetMaxShare());
this.parent = parent;
}
/// <summary>
/// Checks to see whether any other applications runnable now that the given
/// application has been removed from the given queue.
/// </summary>
/// <remarks>
/// Checks to see whether any other applications runnable now that the given
/// application has been removed from the given queue. And makes them so.
/// Runs in O(n log(n)) where n is the number of queues that are under the
/// highest queue that went from having no slack to having slack.
/// </remarks>
public virtual void UpdateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue
)
{
AllocationConfiguration allocConf = scheduler.GetAllocationConfiguration();
// childqueueX might have no pending apps itself, but if a queue higher up
// in the hierarchy parentqueueY has a maxRunningApps set, an app completion
// in childqueueX could allow an app in some other distant child of
// parentqueueY to become runnable.
// An app removal will only possibly allow another app to become runnable if
// the queue was already at its max before the removal.
// Thus we find the ancestor queue highest in the tree for which the app
// that was at its maxRunningApps before the removal.
FSQueue highestQueueWithAppsNowRunnable = (queue.GetNumRunnableApps() == allocConf
.GetQueueMaxApps(queue.GetName()) - 1) ? queue : null;
FSParentQueue parent = queue.GetParent();
while (parent != null)
{
if (parent.GetNumRunnableApps() == allocConf.GetQueueMaxApps(parent.GetName()) -
1)
{
highestQueueWithAppsNowRunnable = parent;
}
parent = parent.GetParent();
}
IList <IList <FSAppAttempt> > appsNowMaybeRunnable = new AList <IList <FSAppAttempt> >(
);
// Compile lists of apps which may now be runnable
// We gather lists instead of building a set of all non-runnable apps so
// that this whole operation can be O(number of queues) instead of
// O(number of apps)
if (highestQueueWithAppsNowRunnable != null)
{
GatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable
);
}
string user = app.GetUser();
int userNumRunning = usersNumRunnableApps[user];
if (userNumRunning == null)
{
userNumRunning = 0;
}
if (userNumRunning == allocConf.GetUserMaxApps(user) - 1)
{
IList <FSAppAttempt> userWaitingApps = usersNonRunnableApps.Get(user);
if (userWaitingApps != null)
{
appsNowMaybeRunnable.AddItem(userWaitingApps);
}
}
UpdateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.Count);
}
public FSLeafQueue(string name, FairScheduler scheduler, FSParentQueue parent)
: base(name, scheduler, parent)
{
// apps that are runnable
// get a lock with fair distribution for app list updates
// Variables used for preemption
// Track the AM resource usage for this queue
this.lastTimeAtMinShare = scheduler.GetClock().GetTime();
this.lastTimeAtFairShareThreshold = scheduler.GetClock().GetTime();
activeUsersManager = new ActiveUsersManager(GetMetrics());
amResourceUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(0, 0);
}
/// <summary>
/// Tracks the given new runnable app for purposes of maintaining max running
/// app limits.
/// </summary>
public virtual void TrackRunnableApp(FSAppAttempt app)
{
string user = app.GetUser();
FSLeafQueue queue = ((FSLeafQueue)app.GetQueue());
// Increment running counts for all parent queues
FSParentQueue parent = queue.GetParent();
while (parent != null)
{
parent.IncrementRunnableApps();
parent = parent.GetParent();
}
int userNumRunnable = usersNumRunnableApps[user];
usersNumRunnableApps[user] = (userNumRunnable == null ? 0 : userNumRunnable) + 1;
}
/// <summary>
/// Updates the relevant tracking variables after a runnable app with the given
/// queue and user has been removed.
/// </summary>
public virtual void UntrackRunnableApp(FSAppAttempt app)
{
// Update usersRunnableApps
string user = app.GetUser();
int newUserNumRunning = usersNumRunnableApps[user] - 1;
if (newUserNumRunning == 0)
{
Sharpen.Collections.Remove(usersNumRunnableApps, user);
}
else
{
usersNumRunnableApps[user] = newUserNumRunning;
}
// Update runnable app bookkeeping for queues
FSLeafQueue queue = ((FSLeafQueue)app.GetQueue());
FSParentQueue parent = queue.GetParent();
while (parent != null)
{
parent.DecrementRunnableApps();
parent = parent.GetParent();
}
}
/// <summary>
/// Creates a leaf or parent queue based on what is specified in 'queueType'
/// and places it in the tree.
/// </summary>
/// <remarks>
/// Creates a leaf or parent queue based on what is specified in 'queueType'
/// and places it in the tree. Creates any parents that don't already exist.
/// </remarks>
/// <returns>
/// the created queue, if successful. null if not allowed (one of the parent
/// queues in the queue name is already a leaf queue)
/// </returns>
private FSQueue CreateQueue(string name, FSQueueType queueType)
{
IList <string> newQueueNames = new AList <string>();
newQueueNames.AddItem(name);
int sepIndex = name.Length;
FSParentQueue parent = null;
// Move up the queue tree until we reach one that exists.
while (sepIndex != -1)
{
sepIndex = name.LastIndexOf('.', sepIndex - 1);
FSQueue queue;
string curName = null;
curName = Sharpen.Runtime.Substring(name, 0, sepIndex);
queue = queues[curName];
if (queue == null)
{
newQueueNames.AddItem(curName);
}
else
{
if (queue is FSParentQueue)
{
parent = (FSParentQueue)queue;
break;
}
else
{
return(null);
}
}
}
// At this point, parent refers to the deepest existing parent of the
// queue to create.
// Now that we know everything worked out, make all the queues
// and add them to the map.
AllocationConfiguration queueConf = scheduler.GetAllocationConfiguration();
FSLeafQueue leafQueue = null;
for (int i = newQueueNames.Count - 1; i >= 0; i--)
{
string queueName = newQueueNames[i];
if (i == 0 && queueType != FSQueueType.Parent)
{
leafQueue = new FSLeafQueue(name, scheduler, parent);
try
{
leafQueue.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
}
catch (AllocationConfigurationException ex)
{
Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
() + " on new leaf queue.", ex);
}
parent.AddChildQueue(leafQueue);
queues[leafQueue.GetName()] = leafQueue;
leafQueues.AddItem(leafQueue);
leafQueue.UpdatePreemptionVariables();
return(leafQueue);
}
else
{
FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent);
try
{
newParent.SetPolicy(queueConf.GetDefaultSchedulingPolicy());
}
catch (AllocationConfigurationException ex)
{
Log.Warn("Failed to set default scheduling policy " + queueConf.GetDefaultSchedulingPolicy
() + " on new parent queue.", ex);
}
parent.AddChildQueue(newParent);
queues[newParent.GetName()] = newParent;
newParent.UpdatePreemptionVariables();
parent = newParent;
}
}
return(parent);
}