public AllocationConfiguration(IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
int> userMaxApps, IDictionary <string, ResourceWeights> queueWeights, IDictionary
<string, float> queueMaxAMShares, int userMaxAppsDefault, int queueMaxAppsDefault
, float queueMaxAMShareDefault, IDictionary <string, SchedulingPolicy> schedulingPolicies
, SchedulingPolicy defaultSchedulingPolicy, IDictionary <string, long> minSharePreemptionTimeouts
, IDictionary <string, long> fairSharePreemptionTimeouts, IDictionary <string, float
> fairSharePreemptionThresholds, IDictionary <string, IDictionary <QueueACL, AccessControlList
> > queueAcls, QueuePlacementPolicy placementPolicy, IDictionary <FSQueueType, ICollection
<string> > configuredQueues, ReservationQueueConfiguration globalReservationQueueConfig
, ICollection <string> reservableQueues)
{
// Minimum resource allocation for each queue
// Maximum amount of resources per queue
// Sharing weights for each queue
// Max concurrent running applications for each queue and for each user; in addition,
// for users that have no max specified, we use the userMaxJobsDefault.
// Maximum resource share for each leaf queue that can be used to run AMs
// ACL's for each queue. Only specifies non-default ACL's from configuration.
// Min share preemption timeout for each queue in seconds. If a job in the queue
// waits this long without receiving its guaranteed share, it is allowed to
// preempt other jobs' tasks.
// Fair share preemption timeout for each queue in seconds. If a job in the
// queue waits this long without receiving its fair share threshold, it is
// allowed to preempt other jobs' tasks.
// The fair share preemption threshold for each queue. If a queue waits
// fairSharePreemptionTimeout without receiving
// fairshare * fairSharePreemptionThreshold resources, it is allowed to
// preempt other queues' tasks.
// Policy for mapping apps to queues
//Configured queues in the alloc xml
// Reservation system configuration
this.minQueueResources = minQueueResources;
this.maxQueueResources = maxQueueResources;
this.queueMaxApps = queueMaxApps;
this.userMaxApps = userMaxApps;
this.queueMaxAMShares = queueMaxAMShares;
this.queueWeights = queueWeights;
this.userMaxAppsDefault = userMaxAppsDefault;
this.queueMaxAppsDefault = queueMaxAppsDefault;
this.queueMaxAMShareDefault = queueMaxAMShareDefault;
this.defaultSchedulingPolicy = defaultSchedulingPolicy;
this.schedulingPolicies = schedulingPolicies;
this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
this.fairSharePreemptionTimeouts = fairSharePreemptionTimeouts;
this.fairSharePreemptionThresholds = fairSharePreemptionThresholds;
this.queueAcls = queueAcls;
this.reservableQueues = reservableQueues;
this.globalReservationQueueConfig = globalReservationQueueConfig;
this.placementPolicy = placementPolicy;
this.configuredQueues = configuredQueues;
}
/// <summary>Updates the allocation list from the allocation config file.</summary>
/// <remarks>
/// Updates the allocation list from the allocation config file. This file is
/// expected to be in the XML format specified in the design doc.
/// </remarks>
/// <exception cref="System.IO.IOException">if the config file cannot be read.</exception>
/// <exception cref="AllocationConfigurationException">if allocations are invalid.</exception>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException">if XML parser is misconfigured.
/// </exception>
/// <exception cref="Org.Xml.Sax.SAXException">if config file is malformed.</exception>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
public virtual void ReloadAllocations()
{
lock (this)
{
if (allocFile == null)
{
return;
}
Log.Info("Loading allocation file " + allocFile);
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
IDictionary <string, Resource> minQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, Resource> maxQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, int> queueMaxApps = new Dictionary <string, int>();
IDictionary <string, int> userMaxApps = new Dictionary <string, int>();
IDictionary <string, float> queueMaxAMShares = new Dictionary <string, float>();
IDictionary <string, ResourceWeights> queueWeights = new Dictionary <string, ResourceWeights
>();
IDictionary <string, SchedulingPolicy> queuePolicies = new Dictionary <string, SchedulingPolicy
>();
IDictionary <string, long> minSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, long> fairSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, float> fairSharePreemptionThresholds = new Dictionary <string,
float>();
IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls = new Dictionary
<string, IDictionary <QueueACL, AccessControlList> >();
ICollection <string> reservableQueues = new HashSet <string>();
int userMaxAppsDefault = int.MaxValue;
int queueMaxAppsDefault = int.MaxValue;
float queueMaxAMShareDefault = 0.5f;
long defaultFairSharePreemptionTimeout = long.MaxValue;
long defaultMinSharePreemptionTimeout = long.MaxValue;
float defaultFairSharePreemptionThreshold = 0.5f;
SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DefaultPolicy;
// Reservation global configuration knobs
string planner = null;
string reservationAgent = null;
string reservationAdmissionPolicy = null;
QueuePlacementPolicy newPlacementPolicy = null;
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
// and also for making queue placement decisions(QueuePlacementRule.java).
IDictionary <FSQueueType, ICollection <string> > configuredQueues = new Dictionary <FSQueueType
, ICollection <string> >();
foreach (FSQueueType queueType in FSQueueType.Values())
{
configuredQueues[queueType] = new HashSet <string>();
}
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
docBuilderFactory.SetIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(allocFile);
Element root = doc.GetDocumentElement();
if (!"allocations".Equals(root.GetTagName()))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>"
);
}
NodeList elements = root.GetChildNodes();
IList <Element> queueElements = new AList <Element>();
Element placementPolicyElement = null;
for (int i = 0; i < elements.GetLength(); i++)
{
Node node = elements.Item(i);
if (node is Element)
{
Element element = (Element)node;
if ("queue".Equals(element.GetTagName()) || "pool".Equals(element.GetTagName()))
{
queueElements.AddItem(element);
}
else
{
if ("user".Equals(element.GetTagName()))
{
string userName = element.GetAttribute("name");
NodeList fields = element.GetChildNodes();
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxApps[userName] = val;
}
}
}
else
{
if ("userMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxAppsDefault = val;
}
else
{
if ("defaultFairSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(element.GetTagName()))
{
if (defaultFairSharePreemptionTimeout == long.MaxValue)
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
}
else
{
if ("defaultMinSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultMinSharePreemptionTimeout = val;
}
else
{
if ("defaultFairSharePreemptionThreshold".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
defaultFairSharePreemptionThreshold = val;
}
else
{
if ("queueMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxAppsDefault = val;
}
else
{
if ("queueMaxAMShareDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShareDefault = val;
}
else
{
if ("defaultQueueSchedulingPolicy".Equals(element.GetTagName()) || "defaultQueueSchedulingMode"
.Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
defaultSchedPolicy = SchedulingPolicy.Parse(text);
}
else
{
if ("queuePlacementPolicy".Equals(element.GetTagName()))
{
placementPolicyElement = element;
}
else
{
if ("reservation-planner".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
planner = text;
}
else
{
if ("reservation-agent".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAgent = text;
}
else
{
if ("reservation-policy".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAdmissionPolicy = text;
}
else
{
Log.Warn("Bad element in allocations file: " + element.GetTagName());
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
// Load queue elements. A root queue can either be included or omitted. If
// it's included, all other queues must be inside it.
foreach (Element element_1 in queueElements)
{
string parent = "root";
if (Sharpen.Runtime.EqualsIgnoreCase(element_1.GetAttribute("name"), "root"))
{
if (queueElements.Count > 1)
{
throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it."
);
}
parent = null;
}
LoadQueue(parent, element_1, minQueueResources, maxQueueResources, queueMaxApps,
userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts
, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues
, reservableQueues);
}
// Load placement policy and pass it configured queues
Configuration conf = GetConfig();
if (placementPolicyElement != null)
{
newPlacementPolicy = QueuePlacementPolicy.FromXml(placementPolicyElement, configuredQueues
, conf);
}
else
{
newPlacementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues
);
}
// Set the min/fair share preemption timeout for the root queue
if (!minSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
minSharePreemptionTimeouts[QueueManager.RootQueue] = defaultMinSharePreemptionTimeout;
}
if (!fairSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
fairSharePreemptionTimeouts[QueueManager.RootQueue] = defaultFairSharePreemptionTimeout;
}
// Set the fair share preemption threshold for the root queue
if (!fairSharePreemptionThresholds.Contains(QueueManager.RootQueue))
{
fairSharePreemptionThresholds[QueueManager.RootQueue] = defaultFairSharePreemptionThreshold;
}
ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration
();
if (planner != null)
{
globalReservationQueueConfig.SetPlanner(planner);
}
if (reservationAdmissionPolicy != null)
{
globalReservationQueueConfig.SetReservationAdmissionPolicy(reservationAdmissionPolicy
);
}
if (reservationAgent != null)
{
globalReservationQueueConfig.SetReservationAgent(reservationAgent);
}
AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources
, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault,
queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy,
minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds
, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig,
reservableQueues);
lastSuccessfulReload = clock.GetTime();
lastReloadAttemptFailed = false;
reloadListener.OnReload(info);
}
}