/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
public virtual void TestParseSchedulingPolicy()
{
// Class name
SchedulingPolicy sm = SchedulingPolicy.Parse(typeof(FairSharePolicy).FullName);
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
.Name));
// Canonical name
sm = SchedulingPolicy.Parse(typeof(FairSharePolicy).GetCanonicalName());
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
.Name));
// Class
sm = SchedulingPolicy.GetInstance(typeof(FairSharePolicy));
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
.Name));
// Shortname - drf
sm = SchedulingPolicy.Parse("drf");
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(DominantResourceFairnessPolicy
.Name));
// Shortname - fair
sm = SchedulingPolicy.Parse("fair");
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
.Name));
// Shortname - fifo
sm = SchedulingPolicy.Parse("fifo");
NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FifoPolicy
.Name));
}
/// <summary>
/// Trivial tests that make sure
/// <see cref="SchedulingPolicy.IsApplicableTo(SchedulingPolicy, byte)"/>
/// works as
/// expected for the possible values of depth
/// </summary>
/// <exception cref="AllocationConfigurationException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
public virtual void TestIsApplicableTo()
{
string Err = "Broken SchedulingPolicy#isApplicableTo";
// fifo
SchedulingPolicy policy = SchedulingPolicy.Parse("fifo");
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthLeaf));
NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(SchedulingPolicy
.Parse("fifo"), SchedulingPolicy.DepthIntermediate));
NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(SchedulingPolicy
.Parse("fifo"), SchedulingPolicy.DepthRoot));
// fair
policy = SchedulingPolicy.Parse("fair");
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthLeaf));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthIntermediate));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthRoot));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthParent));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthAny));
// drf
policy = SchedulingPolicy.Parse("drf");
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthLeaf));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthIntermediate));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthRoot));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthParent));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthAny));
policy = Org.Mockito.Mockito.Mock <SchedulingPolicy>();
Org.Mockito.Mockito.When(policy.GetApplicableDepth()).ThenReturn(SchedulingPolicy
.DepthParent);
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthIntermediate));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthRoot));
NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthParent));
NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
.DepthAny));
}
/// <summary>Loads a queue from a queue element in the configuration file</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
private void LoadQueue(string parentName, Element element, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
int> userMaxApps, IDictionary <string, float> queueMaxAMShares, IDictionary <string
, ResourceWeights> queueWeights, IDictionary <string, SchedulingPolicy> queuePolicies
, IDictionary <string, long> minSharePreemptionTimeouts, IDictionary <string, long
> fairSharePreemptionTimeouts, IDictionary <string, float> fairSharePreemptionThresholds
, IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls, IDictionary
<FSQueueType, ICollection <string> > configuredQueues, ICollection <string> reservableQueues
)
{
string queueName = element.GetAttribute("name");
if (queueName.Contains("."))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name ("
+ queueName + ") shouldn't contain period.");
}
if (parentName != null)
{
queueName = parentName + "." + queueName;
}
IDictionary <QueueACL, AccessControlList> acls = new Dictionary <QueueACL, AccessControlList
>();
NodeList fields = element.GetChildNodes();
bool isLeaf = true;
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("minResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
minQueueResources[queueName] = val;
}
else
{
if ("maxResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
maxQueueResources[queueName] = val;
}
else
{
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxApps[queueName] = val;
}
else
{
if ("maxAMShare".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShares[queueName] = val;
}
else
{
if ("weight".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
double val = double.ParseDouble(text);
queueWeights[queueName] = new ResourceWeights((float)val);
}
else
{
if ("minSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
minSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
fairSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionThreshold".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
fairSharePreemptionThresholds[queueName] = val;
}
else
{
if ("schedulingPolicy".Equals(field.GetTagName()) || "schedulingMode".Equals(field
.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
SchedulingPolicy policy = SchedulingPolicy.Parse(text);
queuePolicies[queueName] = policy;
}
else
{
if ("aclSubmitApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.SubmitApplications] = new AccessControlList(text);
}
else
{
if ("aclAdministerApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.AdministerQueue] = new AccessControlList(text);
}
else
{
if ("reservation".Equals(field.GetTagName()))
{
isLeaf = false;
reservableQueues.AddItem(queueName);
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
if ("queue".EndsWith(field.GetTagName()) || "pool".Equals(field.GetTagName()))
{
LoadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps
, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts
, fairSharePreemptionThresholds, queueAcls, configuredQueues, reservableQueues);
isLeaf = false;
}
}
}
}
}
}
}
}
}
}
}
}
}
}
if (isLeaf)
{
// if a leaf in the alloc file is marked as type='parent'
// then store it under 'parent'
if ("parent".Equals(element.GetAttribute("type")))
{
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
configuredQueues[FSQueueType.Leaf].AddItem(queueName);
}
}
else
{
if ("parent".Equals(element.GetAttribute("type")))
{
throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue "
+ queueName + " which is " + "unsupported");
}
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
queueAcls[queueName] = acls;
if (maxQueueResources.Contains(queueName) && minQueueResources.Contains(queueName
) && !Resources.FitsIn(minQueueResources[queueName], maxQueueResources[queueName
]))
{
Log.Warn(string.Format("Queue %s has max resources %s less than min resources %s"
, queueName, maxQueueResources[queueName], minQueueResources[queueName]));
}
}
/// <summary>Updates the allocation list from the allocation config file.</summary>
/// <remarks>
/// Updates the allocation list from the allocation config file. This file is
/// expected to be in the XML format specified in the design doc.
/// </remarks>
/// <exception cref="System.IO.IOException">if the config file cannot be read.</exception>
/// <exception cref="AllocationConfigurationException">if allocations are invalid.</exception>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException">if XML parser is misconfigured.
/// </exception>
/// <exception cref="Org.Xml.Sax.SAXException">if config file is malformed.</exception>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
public virtual void ReloadAllocations()
{
lock (this)
{
if (allocFile == null)
{
return;
}
Log.Info("Loading allocation file " + allocFile);
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
IDictionary <string, Resource> minQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, Resource> maxQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, int> queueMaxApps = new Dictionary <string, int>();
IDictionary <string, int> userMaxApps = new Dictionary <string, int>();
IDictionary <string, float> queueMaxAMShares = new Dictionary <string, float>();
IDictionary <string, ResourceWeights> queueWeights = new Dictionary <string, ResourceWeights
>();
IDictionary <string, SchedulingPolicy> queuePolicies = new Dictionary <string, SchedulingPolicy
>();
IDictionary <string, long> minSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, long> fairSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, float> fairSharePreemptionThresholds = new Dictionary <string,
float>();
IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls = new Dictionary
<string, IDictionary <QueueACL, AccessControlList> >();
ICollection <string> reservableQueues = new HashSet <string>();
int userMaxAppsDefault = int.MaxValue;
int queueMaxAppsDefault = int.MaxValue;
float queueMaxAMShareDefault = 0.5f;
long defaultFairSharePreemptionTimeout = long.MaxValue;
long defaultMinSharePreemptionTimeout = long.MaxValue;
float defaultFairSharePreemptionThreshold = 0.5f;
SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DefaultPolicy;
// Reservation global configuration knobs
string planner = null;
string reservationAgent = null;
string reservationAdmissionPolicy = null;
QueuePlacementPolicy newPlacementPolicy = null;
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
// and also for making queue placement decisions(QueuePlacementRule.java).
IDictionary <FSQueueType, ICollection <string> > configuredQueues = new Dictionary <FSQueueType
, ICollection <string> >();
foreach (FSQueueType queueType in FSQueueType.Values())
{
configuredQueues[queueType] = new HashSet <string>();
}
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
docBuilderFactory.SetIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(allocFile);
Element root = doc.GetDocumentElement();
if (!"allocations".Equals(root.GetTagName()))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>"
);
}
NodeList elements = root.GetChildNodes();
IList <Element> queueElements = new AList <Element>();
Element placementPolicyElement = null;
for (int i = 0; i < elements.GetLength(); i++)
{
Node node = elements.Item(i);
if (node is Element)
{
Element element = (Element)node;
if ("queue".Equals(element.GetTagName()) || "pool".Equals(element.GetTagName()))
{
queueElements.AddItem(element);
}
else
{
if ("user".Equals(element.GetTagName()))
{
string userName = element.GetAttribute("name");
NodeList fields = element.GetChildNodes();
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxApps[userName] = val;
}
}
}
else
{
if ("userMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxAppsDefault = val;
}
else
{
if ("defaultFairSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(element.GetTagName()))
{
if (defaultFairSharePreemptionTimeout == long.MaxValue)
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
}
else
{
if ("defaultMinSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultMinSharePreemptionTimeout = val;
}
else
{
if ("defaultFairSharePreemptionThreshold".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
defaultFairSharePreemptionThreshold = val;
}
else
{
if ("queueMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxAppsDefault = val;
}
else
{
if ("queueMaxAMShareDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShareDefault = val;
}
else
{
if ("defaultQueueSchedulingPolicy".Equals(element.GetTagName()) || "defaultQueueSchedulingMode"
.Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
defaultSchedPolicy = SchedulingPolicy.Parse(text);
}
else
{
if ("queuePlacementPolicy".Equals(element.GetTagName()))
{
placementPolicyElement = element;
}
else
{
if ("reservation-planner".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
planner = text;
}
else
{
if ("reservation-agent".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAgent = text;
}
else
{
if ("reservation-policy".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAdmissionPolicy = text;
}
else
{
Log.Warn("Bad element in allocations file: " + element.GetTagName());
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
// Load queue elements. A root queue can either be included or omitted. If
// it's included, all other queues must be inside it.
foreach (Element element_1 in queueElements)
{
string parent = "root";
if (Sharpen.Runtime.EqualsIgnoreCase(element_1.GetAttribute("name"), "root"))
{
if (queueElements.Count > 1)
{
throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it."
);
}
parent = null;
}
LoadQueue(parent, element_1, minQueueResources, maxQueueResources, queueMaxApps,
userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts
, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues
, reservableQueues);
}
// Load placement policy and pass it configured queues
Configuration conf = GetConfig();
if (placementPolicyElement != null)
{
newPlacementPolicy = QueuePlacementPolicy.FromXml(placementPolicyElement, configuredQueues
, conf);
}
else
{
newPlacementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues
);
}
// Set the min/fair share preemption timeout for the root queue
if (!minSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
minSharePreemptionTimeouts[QueueManager.RootQueue] = defaultMinSharePreemptionTimeout;
}
if (!fairSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
fairSharePreemptionTimeouts[QueueManager.RootQueue] = defaultFairSharePreemptionTimeout;
}
// Set the fair share preemption threshold for the root queue
if (!fairSharePreemptionThresholds.Contains(QueueManager.RootQueue))
{
fairSharePreemptionThresholds[QueueManager.RootQueue] = defaultFairSharePreemptionThreshold;
}
ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration
();
if (planner != null)
{
globalReservationQueueConfig.SetPlanner(planner);
}
if (reservationAdmissionPolicy != null)
{
globalReservationQueueConfig.SetReservationAdmissionPolicy(reservationAdmissionPolicy
);
}
if (reservationAgent != null)
{
globalReservationQueueConfig.SetReservationAgent(reservationAgent);
}
AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources
, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault,
queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy,
minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds
, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig,
reservableQueues);
lastSuccessfulReload = clock.GetTime();
lastReloadAttemptFailed = false;
reloadListener.OnReload(info);
}
}
