/// <summary>Parse xml and instantiate the nested rule</summary> /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException /// "/> public override void InitializeFromXml(Element el) { NodeList elements = el.GetChildNodes(); for (int i = 0; i < elements.GetLength(); i++) { Node node = elements.Item(i); if (node is Element) { Element element = (Element)node; if ("rule".Equals(element.GetTagName())) { QueuePlacementRule rule = QueuePlacementPolicy.CreateAndInitializeRule(node); if (rule == null) { throw new AllocationConfigurationException("Unable to create nested rule in nestedUserQueue rule" ); } this.nestedRule = rule; break; } else { continue; } } } if (this.nestedRule == null) { throw new AllocationConfigurationException("No nested rule specified in <nestedUserQueue> rule" ); } base.InitializeFromXml(el); }
public virtual void TestNestedUserQueuePrimaryGroup() { StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' create='false' />"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='primaryGroup'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); // User queue would be created under primary group queue QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.user1group.user1", policy.AssignAppToQueue( "root.default", "user1")); // Other rules above and below hierarchical user queue rule should work as // usual configuredQueues[FSQueueType.Leaf].AddItem("root.specifiedq"); // test if specified rule(above nestedUserQueue rule) works ok NUnit.Framework.Assert.AreEqual("root.specifiedq", policy.AssignAppToQueue("root.specifiedq" , "user2")); // test if default rule(below nestedUserQueue rule) works configuredQueues[FSQueueType.Leaf].AddItem("root.user3group"); NUnit.Framework.Assert.AreEqual("root.default", policy.AssignAppToQueue("root.default" , "user3")); }
public AllocationConfiguration(Configuration conf) { minQueueResources = new Dictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource >(); maxQueueResources = new Dictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource >(); queueWeights = new Dictionary <string, ResourceWeights>(); queueMaxApps = new Dictionary <string, int>(); userMaxApps = new Dictionary <string, int>(); queueMaxAMShares = new Dictionary <string, float>(); userMaxAppsDefault = int.MaxValue; queueMaxAppsDefault = int.MaxValue; queueMaxAMShareDefault = 0.5f; queueAcls = new Dictionary <string, IDictionary <QueueACL, AccessControlList> >(); minSharePreemptionTimeouts = new Dictionary <string, long>(); fairSharePreemptionTimeouts = new Dictionary <string, long>(); fairSharePreemptionThresholds = new Dictionary <string, float>(); schedulingPolicies = new Dictionary <string, SchedulingPolicy>(); defaultSchedulingPolicy = SchedulingPolicy.DefaultPolicy; reservableQueues = new HashSet <string>(); configuredQueues = new Dictionary <FSQueueType, ICollection <string> >(); foreach (FSQueueType queueType in FSQueueType.Values()) { configuredQueues[queueType] = new HashSet <string>(); } placementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues); }
public virtual void TestSimplePlacementPolicyFromConf() { Configuration conf = new Configuration(); conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); conf.SetBoolean(FairSchedulerConfiguration.AllowUndeclaredPools, false); conf.SetBoolean(FairSchedulerConfiguration.UserAsDefaultQueue, false); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("</allocations>"); @out.Close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.Init(conf); TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener (this); allocLoader.SetReloadListener(confHolder); allocLoader.ReloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; QueuePlacementPolicy placementPolicy = allocConf.GetPlacementPolicy(); IList <QueuePlacementRule> rules = placementPolicy.GetRules(); NUnit.Framework.Assert.AreEqual(2, rules.Count); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Specified), rules[0].GetType ()); NUnit.Framework.Assert.AreEqual(false, rules[0].create); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Default), rules[1].GetType ()); }
/// <exception cref="System.Exception"/> private QueuePlacementPolicy Parse(string str) { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance(); docBuilderFactory.SetIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder(); Document doc = builder.Parse(IOUtils.ToInputStream(str)); Element root = doc.GetDocumentElement(); return(QueuePlacementPolicy.FromXml(root, configuredQueues, conf)); }
public AllocationConfiguration(IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource > minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource > maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string, int> userMaxApps, IDictionary <string, ResourceWeights> queueWeights, IDictionary <string, float> queueMaxAMShares, int userMaxAppsDefault, int queueMaxAppsDefault , float queueMaxAMShareDefault, IDictionary <string, SchedulingPolicy> schedulingPolicies , SchedulingPolicy defaultSchedulingPolicy, IDictionary <string, long> minSharePreemptionTimeouts , IDictionary <string, long> fairSharePreemptionTimeouts, IDictionary <string, float > fairSharePreemptionThresholds, IDictionary <string, IDictionary <QueueACL, AccessControlList > > queueAcls, QueuePlacementPolicy placementPolicy, IDictionary <FSQueueType, ICollection <string> > configuredQueues, ReservationQueueConfiguration globalReservationQueueConfig , ICollection <string> reservableQueues) { // Minimum resource allocation for each queue // Maximum amount of resources per queue // Sharing weights for each queue // Max concurrent running applications for each queue and for each user; in addition, // for users that have no max specified, we use the userMaxJobsDefault. // Maximum resource share for each leaf queue that can be used to run AMs // ACL's for each queue. Only specifies non-default ACL's from configuration. // Min share preemption timeout for each queue in seconds. If a job in the queue // waits this long without receiving its guaranteed share, it is allowed to // preempt other jobs' tasks. // Fair share preemption timeout for each queue in seconds. If a job in the // queue waits this long without receiving its fair share threshold, it is // allowed to preempt other jobs' tasks. // The fair share preemption threshold for each queue. If a queue waits // fairSharePreemptionTimeout without receiving // fairshare * fairSharePreemptionThreshold resources, it is allowed to // preempt other queues' tasks. // Policy for mapping apps to queues //Configured queues in the alloc xml // Reservation system configuration this.minQueueResources = minQueueResources; this.maxQueueResources = maxQueueResources; this.queueMaxApps = queueMaxApps; this.userMaxApps = userMaxApps; this.queueMaxAMShares = queueMaxAMShares; this.queueWeights = queueWeights; this.userMaxAppsDefault = userMaxAppsDefault; this.queueMaxAppsDefault = queueMaxAppsDefault; this.queueMaxAMShareDefault = queueMaxAMShareDefault; this.defaultSchedulingPolicy = defaultSchedulingPolicy; this.schedulingPolicies = schedulingPolicies; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.fairSharePreemptionTimeouts = fairSharePreemptionTimeouts; this.fairSharePreemptionThresholds = fairSharePreemptionThresholds; this.queueAcls = queueAcls; this.reservableQueues = reservableQueues; this.globalReservationQueueConfig = globalReservationQueueConfig; this.placementPolicy = placementPolicy; this.configuredQueues = configuredQueues; }
public virtual void TestSpecifiedThenReject() { StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' />"); sb.Append(" <rule name='reject' />"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.specifiedq", policy.AssignAppToQueue("specifiedq" , "someuser")); NUnit.Framework.Assert.AreEqual(null, policy.AssignAppToQueue("default", "someuser" )); }
public virtual void TestDefaultRuleWithQueueAttribute() { // This test covers the use case where we would like default rule // to point to a different queue by default rather than root.default configuredQueues[FSQueueType.Leaf].AddItem("root.someDefaultQueue"); StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' create='false' />"); sb.Append(" <rule name='default' queue='root.someDefaultQueue'/>"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.someDefaultQueue", policy.AssignAppToQueue( "root.default", "user1")); }
public virtual void TestNestedUserQueueDefaultRule() { // This test covers the use case where we would like user queues to be // created under a default parent queue configuredQueues[FSQueueType.Parent].AddItem("root.parentq"); StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' create='false' />"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='default' queue='root.parentq'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.parentq.user1", policy.AssignAppToQueue("root.default" , "user1")); }
public virtual void TestNestedUserQueuePrimaryGroupNoCreate() { // Primary group rule has create='false' StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='primaryGroup' create='false'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); // Should return root.default since primary group 'root.user1group' is not // configured NUnit.Framework.Assert.AreEqual("root.default", policy.AssignAppToQueue("root.default" , "user1")); // Let's configure primary group and check if user queue is created configuredQueues[FSQueueType.Parent].AddItem("root.user1group"); policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.user1group.user1", policy.AssignAppToQueue( "root.default", "user1")); // Both Primary group and nestedUserQueue rule has create='false' sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='nestedUserQueue' create='false'>"); sb.Append(" <rule name='primaryGroup' create='false'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); // Should return root.default since primary group and user queue for user 2 // are not configured. NUnit.Framework.Assert.AreEqual("root.default", policy.AssignAppToQueue("root.default" , "user2")); // Now configure both primary group and the user queue for user2 configuredQueues[FSQueueType.Parent].AddItem("root.user2group"); configuredQueues[FSQueueType.Leaf].AddItem("root.user2group.user2"); policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.user2group.user2", policy.AssignAppToQueue( "root.default", "user2")); }
public virtual void TestNoCreate() { StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' />"); sb.Append(" <rule name='user' create=\"false\" />"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); configuredQueues[FSQueueType.Leaf].AddItem("root.someuser"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.specifiedq", policy.AssignAppToQueue("specifiedq" , "someuser")); NUnit.Framework.Assert.AreEqual("root.someuser", policy.AssignAppToQueue("default" , "someuser")); NUnit.Framework.Assert.AreEqual("root.specifiedq", policy.AssignAppToQueue("specifiedq" , "otheruser")); NUnit.Framework.Assert.AreEqual("root.default", policy.AssignAppToQueue("default" , "otheruser")); }
public virtual void TestGroupContainsPeriod() { StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' create='false' />"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='primaryGroup'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(PeriodGroupsMapping ), typeof(GroupMappingServiceProvider)); // User queue would be created under primary group queue, and the period // in the group name should be converted into _dot_ QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.user1_dot_group.user1", policy.AssignAppToQueue ("root.default", "user1")); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping ), typeof(GroupMappingServiceProvider)); }
public virtual void TestNestedUserQueueSpecificRule() { // This test covers the use case where users can specify different parent // queues and want user queues under those. StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='specified' create='false'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); // Let's create couple of parent queues configuredQueues[FSQueueType.Parent].AddItem("root.parent1"); configuredQueues[FSQueueType.Parent].AddItem("root.parent2"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.parent1.user1", policy.AssignAppToQueue("root.parent1" , "user1")); NUnit.Framework.Assert.AreEqual("root.parent2.user2", policy.AssignAppToQueue("root.parent2" , "user2")); }
public virtual void TestNestedUserQueueSecondaryGroup() { StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='secondaryGroupExistingQueue'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); // Should return root.default since secondary groups are not configured NUnit.Framework.Assert.AreEqual("root.default", policy.AssignAppToQueue("root.default" , "user1")); // configure secondary group for user1 configuredQueues[FSQueueType.Parent].AddItem("root.user1subgroup1"); policy = Parse(sb.ToString()); // user queue created should be created under secondary group NUnit.Framework.Assert.AreEqual("root.user1subgroup1.user1", policy.AssignAppToQueue ("root.default", "user1")); }
public virtual void TestUserContainsPeriod() { // This test covers the user case where the username contains periods. StringBuilder sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='user' />"); sb.Append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.first_dot_last", policy.AssignAppToQueue("default" , "first.last")); sb = new StringBuilder(); sb.Append("<queuePlacementPolicy>"); sb.Append(" <rule name='specified' create='false' />"); sb.Append(" <rule name='nestedUserQueue'>"); sb.Append(" <rule name='default'/>"); sb.Append(" </rule>"); sb.Append(" <rule name='default' />"); sb.Append("</queuePlacementPolicy>"); policy = Parse(sb.ToString()); NUnit.Framework.Assert.AreEqual("root.default.first_dot_last", policy.AssignAppToQueue ("root.default", "first.last")); }
/// <summary>Updates the allocation list from the allocation config file.</summary> /// <remarks> /// Updates the allocation list from the allocation config file. This file is /// expected to be in the XML format specified in the design doc. /// </remarks> /// <exception cref="System.IO.IOException">if the config file cannot be read.</exception> /// <exception cref="AllocationConfigurationException">if allocations are invalid.</exception> /// <exception cref="Javax.Xml.Parsers.ParserConfigurationException">if XML parser is misconfigured. /// </exception> /// <exception cref="Org.Xml.Sax.SAXException">if config file is malformed.</exception> /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException /// "/> public virtual void ReloadAllocations() { lock (this) { if (allocFile == null) { return; } Log.Info("Loading allocation file " + allocFile); // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. IDictionary <string, Resource> minQueueResources = new Dictionary <string, Resource >(); IDictionary <string, Resource> maxQueueResources = new Dictionary <string, Resource >(); IDictionary <string, int> queueMaxApps = new Dictionary <string, int>(); IDictionary <string, int> userMaxApps = new Dictionary <string, int>(); IDictionary <string, float> queueMaxAMShares = new Dictionary <string, float>(); IDictionary <string, ResourceWeights> queueWeights = new Dictionary <string, ResourceWeights >(); IDictionary <string, SchedulingPolicy> queuePolicies = new Dictionary <string, SchedulingPolicy >(); IDictionary <string, long> minSharePreemptionTimeouts = new Dictionary <string, long >(); IDictionary <string, long> fairSharePreemptionTimeouts = new Dictionary <string, long >(); IDictionary <string, float> fairSharePreemptionThresholds = new Dictionary <string, float>(); IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls = new Dictionary <string, IDictionary <QueueACL, AccessControlList> >(); ICollection <string> reservableQueues = new HashSet <string>(); int userMaxAppsDefault = int.MaxValue; int queueMaxAppsDefault = int.MaxValue; float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = long.MaxValue; long defaultMinSharePreemptionTimeout = long.MaxValue; float defaultFairSharePreemptionThreshold = 0.5f; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DefaultPolicy; // Reservation global configuration knobs string planner = null; string reservationAgent = null; string reservationAdmissionPolicy = null; QueuePlacementPolicy newPlacementPolicy = null; // Remember all queue names so we can display them on web UI, etc. // configuredQueues is segregated based on whether it is a leaf queue // or a parent queue. This information is used for creating queues // and also for making queue placement decisions(QueuePlacementRule.java). IDictionary <FSQueueType, ICollection <string> > configuredQueues = new Dictionary <FSQueueType , ICollection <string> >(); foreach (FSQueueType queueType in FSQueueType.Values()) { configuredQueues[queueType] = new HashSet <string>(); } // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance(); docBuilderFactory.SetIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder(); Document doc = builder.Parse(allocFile); Element root = doc.GetDocumentElement(); if (!"allocations".Equals(root.GetTagName())) { throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>" ); } NodeList elements = root.GetChildNodes(); IList <Element> queueElements = new AList <Element>(); Element placementPolicyElement = null; for (int i = 0; i < elements.GetLength(); i++) { Node node = elements.Item(i); if (node is Element) { Element element = (Element)node; if ("queue".Equals(element.GetTagName()) || "pool".Equals(element.GetTagName())) { queueElements.AddItem(element); } else { if ("user".Equals(element.GetTagName())) { string userName = element.GetAttribute("name"); NodeList fields = element.GetChildNodes(); for (int j = 0; j < fields.GetLength(); j++) { Node fieldNode = fields.Item(j); if (!(fieldNode is Element)) { continue; } Element field = (Element)fieldNode; if ("maxRunningApps".Equals(field.GetTagName())) { string text = ((Text)field.GetFirstChild()).GetData().Trim(); int val = System.Convert.ToInt32(text); userMaxApps[userName] = val; } } } else { if ("userMaxAppsDefault".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); int val = System.Convert.ToInt32(text); userMaxAppsDefault = val; } else { if ("defaultFairSharePreemptionTimeout".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); long val = long.Parse(text) * 1000L; defaultFairSharePreemptionTimeout = val; } else { if ("fairSharePreemptionTimeout".Equals(element.GetTagName())) { if (defaultFairSharePreemptionTimeout == long.MaxValue) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); long val = long.Parse(text) * 1000L; defaultFairSharePreemptionTimeout = val; } } else { if ("defaultMinSharePreemptionTimeout".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); long val = long.Parse(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else { if ("defaultFairSharePreemptionThreshold".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); float val = float.ParseFloat(text); val = Math.Max(Math.Min(val, 1.0f), 0.0f); defaultFairSharePreemptionThreshold = val; } else { if ("queueMaxAppsDefault".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); int val = System.Convert.ToInt32(text); queueMaxAppsDefault = val; } else { if ("queueMaxAMShareDefault".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); float val = float.ParseFloat(text); val = Math.Min(val, 1.0f); queueMaxAMShareDefault = val; } else { if ("defaultQueueSchedulingPolicy".Equals(element.GetTagName()) || "defaultQueueSchedulingMode" .Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); defaultSchedPolicy = SchedulingPolicy.Parse(text); } else { if ("queuePlacementPolicy".Equals(element.GetTagName())) { placementPolicyElement = element; } else { if ("reservation-planner".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); planner = text; } else { if ("reservation-agent".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); reservationAgent = text; } else { if ("reservation-policy".Equals(element.GetTagName())) { string text = ((Text)element.GetFirstChild()).GetData().Trim(); reservationAdmissionPolicy = text; } else { Log.Warn("Bad element in allocations file: " + element.GetTagName()); } } } } } } } } } } } } } } } } // Load queue elements. A root queue can either be included or omitted. If // it's included, all other queues must be inside it. foreach (Element element_1 in queueElements) { string parent = "root"; if (Sharpen.Runtime.EqualsIgnoreCase(element_1.GetAttribute("name"), "root")) { if (queueElements.Count > 1) { throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it." ); } parent = null; } LoadQueue(parent, element_1, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts , fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues , reservableQueues); } // Load placement policy and pass it configured queues Configuration conf = GetConfig(); if (placementPolicyElement != null) { newPlacementPolicy = QueuePlacementPolicy.FromXml(placementPolicyElement, configuredQueues , conf); } else { newPlacementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues ); } // Set the min/fair share preemption timeout for the root queue if (!minSharePreemptionTimeouts.Contains(QueueManager.RootQueue)) { minSharePreemptionTimeouts[QueueManager.RootQueue] = defaultMinSharePreemptionTimeout; } if (!fairSharePreemptionTimeouts.Contains(QueueManager.RootQueue)) { fairSharePreemptionTimeouts[QueueManager.RootQueue] = defaultFairSharePreemptionTimeout; } // Set the fair share preemption threshold for the root queue if (!fairSharePreemptionThresholds.Contains(QueueManager.RootQueue)) { fairSharePreemptionThresholds[QueueManager.RootQueue] = defaultFairSharePreemptionThreshold; } ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration (); if (planner != null) { globalReservationQueueConfig.SetPlanner(planner); } if (reservationAdmissionPolicy != null) { globalReservationQueueConfig.SetReservationAdmissionPolicy(reservationAdmissionPolicy ); } if (reservationAgent != null) { globalReservationQueueConfig.SetReservationAgent(reservationAgent); } AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources , queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds , queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues); lastSuccessfulReload = clock.GetTime(); lastReloadAttemptFailed = false; reloadListener.OnReload(info); } }
/// <exception cref="System.Exception"/> public virtual void TestReload() { PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine(" <queue name=\"queueA\">"); @out.WriteLine(" <maxRunningApps>1</maxRunningApps>"); @out.WriteLine(" </queue>"); @out.WriteLine(" <queue name=\"queueB\" />"); @out.WriteLine(" <queuePlacementPolicy>"); @out.WriteLine(" <rule name='default' />"); @out.WriteLine(" </queuePlacementPolicy>"); @out.WriteLine("</allocations>"); @out.Close(); TestAllocationFileLoaderService.MockClock clock = new TestAllocationFileLoaderService.MockClock (this); Configuration conf = new Configuration(); conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(clock); allocLoader.reloadIntervalMs = 5; allocLoader.Init(conf); TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener (this); allocLoader.SetReloadListener(confHolder); allocLoader.ReloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; // Verify conf QueuePlacementPolicy policy = allocConf.GetPlacementPolicy(); IList <QueuePlacementRule> rules = policy.GetRules(); NUnit.Framework.Assert.AreEqual(1, rules.Count); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Default), rules[0].GetType ()); NUnit.Framework.Assert.AreEqual(1, allocConf.GetQueueMaxApps("root.queueA")); NUnit.Framework.Assert.AreEqual(2, allocConf.GetConfiguredQueues()[FSQueueType.Leaf ].Count); NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains ("root.queueA")); NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains ("root.queueB")); confHolder.allocConf = null; // Modify file and advance the clock @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine(" <queue name=\"queueB\">"); @out.WriteLine(" <maxRunningApps>3</maxRunningApps>"); @out.WriteLine(" </queue>"); @out.WriteLine(" <queuePlacementPolicy>"); @out.WriteLine(" <rule name='specified' />"); @out.WriteLine(" <rule name='nestedUserQueue' >"); @out.WriteLine(" <rule name='primaryGroup' />"); @out.WriteLine(" </rule>"); @out.WriteLine(" <rule name='default' />"); @out.WriteLine(" </queuePlacementPolicy>"); @out.WriteLine("</allocations>"); @out.Close(); clock.Tick(Runtime.CurrentTimeMillis() + AllocationFileLoaderService.AllocReloadWaitMs + 10000); allocLoader.Start(); while (confHolder.allocConf == null) { Sharpen.Thread.Sleep(20); } // Verify conf allocConf = confHolder.allocConf; policy = allocConf.GetPlacementPolicy(); rules = policy.GetRules(); NUnit.Framework.Assert.AreEqual(3, rules.Count); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Specified), rules[0].GetType ()); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.NestedUserQueue), rules [1].GetType()); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.PrimaryGroup), ((QueuePlacementRule.NestedUserQueue )(rules[1])).nestedRule.GetType()); NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Default), rules[2].GetType ()); NUnit.Framework.Assert.AreEqual(3, allocConf.GetQueueMaxApps("root.queueB")); NUnit.Framework.Assert.AreEqual(1, allocConf.GetConfiguredQueues()[FSQueueType.Leaf ].Count); NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains ("root.queueB")); }