Exemplo n.º 1
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
        ///     "/>
        public virtual void TestParseSchedulingPolicy()
        {
            // Class name
            SchedulingPolicy sm = SchedulingPolicy.Parse(typeof(FairSharePolicy).FullName);

            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
                                                                                        .Name));
            // Canonical name
            sm = SchedulingPolicy.Parse(typeof(FairSharePolicy).GetCanonicalName());
            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
                                                                                        .Name));
            // Class
            sm = SchedulingPolicy.GetInstance(typeof(FairSharePolicy));
            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
                                                                                        .Name));
            // Shortname - drf
            sm = SchedulingPolicy.Parse("drf");
            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(DominantResourceFairnessPolicy
                                                                                        .Name));
            // Shortname - fair
            sm = SchedulingPolicy.Parse("fair");
            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FairSharePolicy
                                                                                        .Name));
            // Shortname - fifo
            sm = SchedulingPolicy.Parse("fifo");
            NUnit.Framework.Assert.IsTrue("Invalid scheduler name", sm.GetName().Equals(FifoPolicy
                                                                                        .Name));
        }
Exemplo n.º 2
0
 public AllocationConfiguration(Configuration conf)
 {
     minQueueResources = new Dictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                         >();
     maxQueueResources = new Dictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                         >();
     queueWeights           = new Dictionary <string, ResourceWeights>();
     queueMaxApps           = new Dictionary <string, int>();
     userMaxApps            = new Dictionary <string, int>();
     queueMaxAMShares       = new Dictionary <string, float>();
     userMaxAppsDefault     = int.MaxValue;
     queueMaxAppsDefault    = int.MaxValue;
     queueMaxAMShareDefault = 0.5f;
     queueAcls = new Dictionary <string, IDictionary <QueueACL, AccessControlList> >();
     minSharePreemptionTimeouts    = new Dictionary <string, long>();
     fairSharePreemptionTimeouts   = new Dictionary <string, long>();
     fairSharePreemptionThresholds = new Dictionary <string, float>();
     schedulingPolicies            = new Dictionary <string, SchedulingPolicy>();
     defaultSchedulingPolicy       = SchedulingPolicy.DefaultPolicy;
     reservableQueues = new HashSet <string>();
     configuredQueues = new Dictionary <FSQueueType, ICollection <string> >();
     foreach (FSQueueType queueType in FSQueueType.Values())
     {
         configuredQueues[queueType] = new HashSet <string>();
     }
     placementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues);
 }
Exemplo n.º 3
0
        /// <summary>
        /// Headroom depends on resources in the cluster, current usage of the
        /// queue, queue's fair-share and queue's max-resources.
        /// </summary>
        public override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetHeadroom()
        {
            FSQueue          queue  = (FSQueue)this.queue;
            SchedulingPolicy policy = queue.GetPolicy();

            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare  = queue.GetFairShare();
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage      = queue.GetResourceUsage();
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = this.scheduler.GetClusterResource
                                                                              ();
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = this.scheduler.GetRootQueueMetrics
                                                                           ().GetAllocatedResources();
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailableResources = Resources
                                                                                    .Subtract(clusterResource, clusterUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxAvailableResources = Resources
                                                                                     .Subtract(queue.GetMaxShare(), queueUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource maxAvailableResource = Resources.ComponentwiseMin
                                                                                   (clusterAvailableResources, queueMaxAvailableResources);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource headroom = policy.GetHeadroom(queueFairShare
                                                                                      , queueUsage, maxAvailableResource);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("Headroom calculation for " + this.GetName() + ":" + "Min(" + "(queueFairShare="
                          + queueFairShare + " - queueUsage=" + queueUsage + ")," + " maxAvailableResource="
                          + maxAvailableResource + "Headroom=" + headroom);
            }
            return(headroom);
        }
Exemplo n.º 4
0
        public virtual void TestHeadroom()
        {
            FairScheduler mockScheduler = Org.Mockito.Mockito.Mock <FairScheduler>();

            Org.Mockito.Mockito.When(mockScheduler.GetClock()).ThenReturn(scheduler.GetClock(
                                                                              ));
            FSLeafQueue mockQueue         = Org.Mockito.Mockito.Mock <FSLeafQueue>();
            Resource    queueMaxResources = Resource.NewInstance(5 * 1024, 3);

            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueFairShare = Resources.CreateResource
                                                                             (4096, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueUsage = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                     .NewInstance(2048, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueStarvation = Resources.Subtract(
                queueFairShare, queueUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueMaxResourcesAvailable = Resources
                                                                                     .Subtract(queueMaxResources, queueUsage);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
                                                                              (8192, 8);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterUsage = Resources.CreateResource
                                                                           (2048, 2);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterAvailable = Resources.Subtract
                                                                               (clusterResource, clusterUsage);
            QueueMetrics fakeRootQueueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();

            Org.Mockito.Mockito.When(mockQueue.GetMaxShare()).ThenReturn(queueMaxResources);
            Org.Mockito.Mockito.When(mockQueue.GetFairShare()).ThenReturn(queueFairShare);
            Org.Mockito.Mockito.When(mockQueue.GetResourceUsage()).ThenReturn(queueUsage);
            Org.Mockito.Mockito.When(mockScheduler.GetClusterResource()).ThenReturn(clusterResource
                                                                                    );
            Org.Mockito.Mockito.When(fakeRootQueueMetrics.GetAllocatedResources()).ThenReturn
                (clusterUsage);
            Org.Mockito.Mockito.When(mockScheduler.GetRootQueueMetrics()).ThenReturn(fakeRootQueueMetrics
                                                                                     );
            ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
            RMContext            rmContext            = resourceManager.GetRMContext();
            FSAppAttempt         schedulerApp         = new FSAppAttempt(mockScheduler, applicationAttemptId,
                                                                         "user1", mockQueue, null, rmContext);

            // Min of Memory and CPU across cluster and queue is used in
            // DominantResourceFairnessPolicy
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(DominantResourceFairnessPolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Min(queueStarvation.GetVirtualCores
                                                                                                      (), clusterAvailable.GetVirtualCores(), queueMaxResourcesAvailable.GetVirtualCores
                                                                                                      ()));
            // Fair and Fifo ignore CPU of queue, so use cluster available CPU
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(FairSharePolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
                                                                                                           (), queueMaxResourcesAvailable.GetVirtualCores()));
            Org.Mockito.Mockito.When(mockQueue.GetPolicy()).ThenReturn(SchedulingPolicy.GetInstance
                                                                           (typeof(FifoPolicy)));
            VerifyHeadroom(schedulerApp, Min(queueStarvation.GetMemory(), clusterAvailable.GetMemory
                                                 (), queueMaxResourcesAvailable.GetMemory()), Math.Min(clusterAvailable.GetVirtualCores
                                                                                                           (), queueMaxResourcesAvailable.GetVirtualCores()));
        }
Exemplo n.º 5
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
 ///     "/>
 public override void SetPolicy(SchedulingPolicy policy)
 {
     if (!SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy.DepthLeaf))
     {
         ThrowPolicyDoesnotApplyException(policy);
     }
     base.policy = policy;
 }
Exemplo n.º 6
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
        ///     "/>
        public override void SetPolicy(SchedulingPolicy policy)
        {
            bool allowed = SchedulingPolicy.IsApplicableTo(policy, (parent == null) ? SchedulingPolicy
                                                           .DepthRoot : SchedulingPolicy.DepthIntermediate);

            if (!allowed)
            {
                ThrowPolicyDoesnotApplyException(policy);
            }
            base.policy = policy;
        }
Exemplo n.º 7
0
        // Root and Intermediate
        /// <summary>
        /// Returns a
        /// <see cref="SchedulingPolicy"/>
        /// instance corresponding to the passed clazz
        /// </summary>
        public static SchedulingPolicy GetInstance(Type clazz)
        {
            SchedulingPolicy policy    = ReflectionUtils.NewInstance(clazz, null);
            SchedulingPolicy policyRet = instances.PutIfAbsent(clazz, policy);

            if (policyRet != null)
            {
                return(policyRet);
            }
            return(policy);
        }
Exemplo n.º 8
0
 public AllocationConfiguration(IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                             > minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                               > maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
                                                                                                                                                         int> userMaxApps, IDictionary <string, ResourceWeights> queueWeights, IDictionary
                                <string, float> queueMaxAMShares, int userMaxAppsDefault, int queueMaxAppsDefault
                                , float queueMaxAMShareDefault, IDictionary <string, SchedulingPolicy> schedulingPolicies
                                , SchedulingPolicy defaultSchedulingPolicy, IDictionary <string, long> minSharePreemptionTimeouts
                                , IDictionary <string, long> fairSharePreemptionTimeouts, IDictionary <string, float
                                                                                                       > fairSharePreemptionThresholds, IDictionary <string, IDictionary <QueueACL, AccessControlList
                                                                                                                                                                          > > queueAcls, QueuePlacementPolicy placementPolicy, IDictionary <FSQueueType, ICollection
                                                                                                                                                                                                                                            <string> > configuredQueues, ReservationQueueConfiguration globalReservationQueueConfig
                                , ICollection <string> reservableQueues)
 {
     // Minimum resource allocation for each queue
     // Maximum amount of resources per queue
     // Sharing weights for each queue
     // Max concurrent running applications for each queue and for each user; in addition,
     // for users that have no max specified, we use the userMaxJobsDefault.
     // Maximum resource share for each leaf queue that can be used to run AMs
     // ACL's for each queue. Only specifies non-default ACL's from configuration.
     // Min share preemption timeout for each queue in seconds. If a job in the queue
     // waits this long without receiving its guaranteed share, it is allowed to
     // preempt other jobs' tasks.
     // Fair share preemption timeout for each queue in seconds. If a job in the
     // queue waits this long without receiving its fair share threshold, it is
     // allowed to preempt other jobs' tasks.
     // The fair share preemption threshold for each queue. If a queue waits
     // fairSharePreemptionTimeout without receiving
     // fairshare * fairSharePreemptionThreshold resources, it is allowed to
     // preempt other queues' tasks.
     // Policy for mapping apps to queues
     //Configured queues in the alloc xml
     // Reservation system configuration
     this.minQueueResources             = minQueueResources;
     this.maxQueueResources             = maxQueueResources;
     this.queueMaxApps                  = queueMaxApps;
     this.userMaxApps                   = userMaxApps;
     this.queueMaxAMShares              = queueMaxAMShares;
     this.queueWeights                  = queueWeights;
     this.userMaxAppsDefault            = userMaxAppsDefault;
     this.queueMaxAppsDefault           = queueMaxAppsDefault;
     this.queueMaxAMShareDefault        = queueMaxAMShareDefault;
     this.defaultSchedulingPolicy       = defaultSchedulingPolicy;
     this.schedulingPolicies            = schedulingPolicies;
     this.minSharePreemptionTimeouts    = minSharePreemptionTimeouts;
     this.fairSharePreemptionTimeouts   = fairSharePreemptionTimeouts;
     this.fairSharePreemptionThresholds = fairSharePreemptionThresholds;
     this.queueAcls                    = queueAcls;
     this.reservableQueues             = reservableQueues;
     this.globalReservationQueueConfig = globalReservationQueueConfig;
     this.placementPolicy              = placementPolicy;
     this.configuredQueues             = configuredQueues;
 }
Exemplo n.º 9
0
        /// <summary>
        /// Trivial tests that make sure
        /// <see cref="SchedulingPolicy.IsApplicableTo(SchedulingPolicy, byte)"/>
        /// works as
        /// expected for the possible values of depth
        /// </summary>
        /// <exception cref="AllocationConfigurationException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
        ///     "/>
        public virtual void TestIsApplicableTo()
        {
            string Err = "Broken SchedulingPolicy#isApplicableTo";
            // fifo
            SchedulingPolicy policy = SchedulingPolicy.Parse("fifo");

            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthLeaf));
            NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(SchedulingPolicy
                                                                                .Parse("fifo"), SchedulingPolicy.DepthIntermediate));
            NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(SchedulingPolicy
                                                                                .Parse("fifo"), SchedulingPolicy.DepthRoot));
            // fair
            policy = SchedulingPolicy.Parse("fair");
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthLeaf));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthIntermediate));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthRoot));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthParent));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthAny));
            // drf
            policy = SchedulingPolicy.Parse("drf");
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthLeaf));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthIntermediate));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthRoot));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthParent));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthAny));
            policy = Org.Mockito.Mockito.Mock <SchedulingPolicy>();
            Org.Mockito.Mockito.When(policy.GetApplicableDepth()).ThenReturn(SchedulingPolicy
                                                                             .DepthParent);
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthIntermediate));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthRoot));
            NUnit.Framework.Assert.IsTrue(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                               .DepthParent));
            NUnit.Framework.Assert.IsFalse(Err, SchedulingPolicy.IsApplicableTo(policy, SchedulingPolicy
                                                                                .DepthAny));
        }
Exemplo n.º 10
0
 public virtual void UpdateAllocationConfiguration(AllocationConfiguration queueConf
                                                   )
 {
     // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist
     foreach (string name in queueConf.GetConfiguredQueues()[FSQueueType.Leaf])
     {
         if (RemoveEmptyIncompatibleQueues(name, FSQueueType.Leaf))
         {
             GetLeafQueue(name, true);
         }
     }
     // At this point all leaves and 'parents with at least one child' would have been created.
     // Now create parents with no configured leaf.
     foreach (string name_1 in queueConf.GetConfiguredQueues()[FSQueueType.Parent])
     {
         if (RemoveEmptyIncompatibleQueues(name_1, FSQueueType.Parent))
         {
             GetParentQueue(name_1, true);
         }
     }
     foreach (FSQueue queue in queues.Values)
     {
         // Update queue metrics
         FSQueueMetrics queueMetrics = queue.GetMetrics();
         queueMetrics.SetMinShare(queue.GetMinShare());
         queueMetrics.SetMaxShare(queue.GetMaxShare());
         // Set scheduling policies
         try
         {
             SchedulingPolicy policy = queueConf.GetSchedulingPolicy(queue.GetName());
             policy.Initialize(scheduler.GetClusterResource());
             queue.SetPolicy(policy);
         }
         catch (AllocationConfigurationException ex)
         {
             Log.Warn("Cannot apply configured scheduling policy to queue " + queue.GetName(),
                      ex);
         }
     }
     // Update steady fair shares for all queues
     rootQueue.RecomputeSteadyShares();
     // Update the fair share preemption timeouts and preemption for all queues
     // recursively
     rootQueue.UpdatePreemptionVariables();
 }
        /// <summary>Loads a queue from a queue element in the configuration file</summary>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
        ///     "/>
        private void LoadQueue(string parentName, Element element, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                                > minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                                                                  > maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
                                                                                                                                                                                            int> userMaxApps, IDictionary <string, float> queueMaxAMShares, IDictionary <string
                                                                                                                                                                                                                                                                         , ResourceWeights> queueWeights, IDictionary <string, SchedulingPolicy> queuePolicies
                               , IDictionary <string, long> minSharePreemptionTimeouts, IDictionary <string, long
                                                                                                     > fairSharePreemptionTimeouts, IDictionary <string, float> fairSharePreemptionThresholds
                               , IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls, IDictionary
                               <FSQueueType, ICollection <string> > configuredQueues, ICollection <string> reservableQueues
                               )
        {
            string queueName = element.GetAttribute("name");

            if (queueName.Contains("."))
            {
                throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name ("
                                                           + queueName + ") shouldn't contain period.");
            }
            if (parentName != null)
            {
                queueName = parentName + "." + queueName;
            }
            IDictionary <QueueACL, AccessControlList> acls = new Dictionary <QueueACL, AccessControlList
                                                                             >();
            NodeList fields = element.GetChildNodes();
            bool     isLeaf = true;

            for (int j = 0; j < fields.GetLength(); j++)
            {
                Node fieldNode = fields.Item(j);
                if (!(fieldNode is Element))
                {
                    continue;
                }
                Element field = (Element)fieldNode;
                if ("minResources".Equals(field.GetTagName()))
                {
                    string text = ((Text)field.GetFirstChild()).GetData().Trim();
                    Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
                                                                          (text);
                    minQueueResources[queueName] = val;
                }
                else
                {
                    if ("maxResources".Equals(field.GetTagName()))
                    {
                        string text = ((Text)field.GetFirstChild()).GetData().Trim();
                        Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
                                                                              (text);
                        maxQueueResources[queueName] = val;
                    }
                    else
                    {
                        if ("maxRunningApps".Equals(field.GetTagName()))
                        {
                            string text = ((Text)field.GetFirstChild()).GetData().Trim();
                            int    val  = System.Convert.ToInt32(text);
                            queueMaxApps[queueName] = val;
                        }
                        else
                        {
                            if ("maxAMShare".Equals(field.GetTagName()))
                            {
                                string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                float  val  = float.ParseFloat(text);
                                val = Math.Min(val, 1.0f);
                                queueMaxAMShares[queueName] = val;
                            }
                            else
                            {
                                if ("weight".Equals(field.GetTagName()))
                                {
                                    string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                    double val  = double.ParseDouble(text);
                                    queueWeights[queueName] = new ResourceWeights((float)val);
                                }
                                else
                                {
                                    if ("minSharePreemptionTimeout".Equals(field.GetTagName()))
                                    {
                                        string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                        long   val  = long.Parse(text) * 1000L;
                                        minSharePreemptionTimeouts[queueName] = val;
                                    }
                                    else
                                    {
                                        if ("fairSharePreemptionTimeout".Equals(field.GetTagName()))
                                        {
                                            string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                            long   val  = long.Parse(text) * 1000L;
                                            fairSharePreemptionTimeouts[queueName] = val;
                                        }
                                        else
                                        {
                                            if ("fairSharePreemptionThreshold".Equals(field.GetTagName()))
                                            {
                                                string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                                float  val  = float.ParseFloat(text);
                                                val = Math.Max(Math.Min(val, 1.0f), 0.0f);
                                                fairSharePreemptionThresholds[queueName] = val;
                                            }
                                            else
                                            {
                                                if ("schedulingPolicy".Equals(field.GetTagName()) || "schedulingMode".Equals(field
                                                                                                                             .GetTagName()))
                                                {
                                                    string           text   = ((Text)field.GetFirstChild()).GetData().Trim();
                                                    SchedulingPolicy policy = SchedulingPolicy.Parse(text);
                                                    queuePolicies[queueName] = policy;
                                                }
                                                else
                                                {
                                                    if ("aclSubmitApps".Equals(field.GetTagName()))
                                                    {
                                                        string text = ((Text)field.GetFirstChild()).GetData();
                                                        acls[QueueACL.SubmitApplications] = new AccessControlList(text);
                                                    }
                                                    else
                                                    {
                                                        if ("aclAdministerApps".Equals(field.GetTagName()))
                                                        {
                                                            string text = ((Text)field.GetFirstChild()).GetData();
                                                            acls[QueueACL.AdministerQueue] = new AccessControlList(text);
                                                        }
                                                        else
                                                        {
                                                            if ("reservation".Equals(field.GetTagName()))
                                                            {
                                                                isLeaf = false;
                                                                reservableQueues.AddItem(queueName);
                                                                configuredQueues[FSQueueType.Parent].AddItem(queueName);
                                                            }
                                                            else
                                                            {
                                                                if ("queue".EndsWith(field.GetTagName()) || "pool".Equals(field.GetTagName()))
                                                                {
                                                                    LoadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps
                                                                              , queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts
                                                                              , fairSharePreemptionThresholds, queueAcls, configuredQueues, reservableQueues);
                                                                    isLeaf = false;
                                                                }
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
            if (isLeaf)
            {
                // if a leaf in the alloc file is marked as type='parent'
                // then store it under 'parent'
                if ("parent".Equals(element.GetAttribute("type")))
                {
                    configuredQueues[FSQueueType.Parent].AddItem(queueName);
                }
                else
                {
                    configuredQueues[FSQueueType.Leaf].AddItem(queueName);
                }
            }
            else
            {
                if ("parent".Equals(element.GetAttribute("type")))
                {
                    throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue "
                                                               + queueName + " which is " + "unsupported");
                }
                configuredQueues[FSQueueType.Parent].AddItem(queueName);
            }
            queueAcls[queueName] = acls;
            if (maxQueueResources.Contains(queueName) && minQueueResources.Contains(queueName
                                                                                    ) && !Resources.FitsIn(minQueueResources[queueName], maxQueueResources[queueName
                                                                                                           ]))
            {
                Log.Warn(string.Format("Queue %s has max resources %s less than min resources %s"
                                       , queueName, maxQueueResources[queueName], minQueueResources[queueName]));
            }
        }
 /// <summary>Updates the allocation list from the allocation config file.</summary>
 /// <remarks>
 /// Updates the allocation list from the allocation config file. This file is
 /// expected to be in the XML format specified in the design doc.
 /// </remarks>
 /// <exception cref="System.IO.IOException">if the config file cannot be read.</exception>
 /// <exception cref="AllocationConfigurationException">if allocations are invalid.</exception>
 /// <exception cref="Javax.Xml.Parsers.ParserConfigurationException">if XML parser is misconfigured.
 ///     </exception>
 /// <exception cref="Org.Xml.Sax.SAXException">if config file is malformed.</exception>
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
 ///     "/>
 public virtual void ReloadAllocations()
 {
     lock (this)
     {
         if (allocFile == null)
         {
             return;
         }
         Log.Info("Loading allocation file " + allocFile);
         // Create some temporary hashmaps to hold the new allocs, and we only save
         // them in our fields if we have parsed the entire allocs file successfully.
         IDictionary <string, Resource> minQueueResources = new Dictionary <string, Resource
                                                                            >();
         IDictionary <string, Resource> maxQueueResources = new Dictionary <string, Resource
                                                                            >();
         IDictionary <string, int>             queueMaxApps     = new Dictionary <string, int>();
         IDictionary <string, int>             userMaxApps      = new Dictionary <string, int>();
         IDictionary <string, float>           queueMaxAMShares = new Dictionary <string, float>();
         IDictionary <string, ResourceWeights> queueWeights     = new Dictionary <string, ResourceWeights
                                                                                  >();
         IDictionary <string, SchedulingPolicy> queuePolicies = new Dictionary <string, SchedulingPolicy
                                                                                >();
         IDictionary <string, long> minSharePreemptionTimeouts = new Dictionary <string, long
                                                                                 >();
         IDictionary <string, long> fairSharePreemptionTimeouts = new Dictionary <string, long
                                                                                  >();
         IDictionary <string, float> fairSharePreemptionThresholds = new Dictionary <string,
                                                                                     float>();
         IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls = new Dictionary
                                                                                      <string, IDictionary <QueueACL, AccessControlList> >();
         ICollection <string> reservableQueues                = new HashSet <string>();
         int              userMaxAppsDefault                  = int.MaxValue;
         int              queueMaxAppsDefault                 = int.MaxValue;
         float            queueMaxAMShareDefault              = 0.5f;
         long             defaultFairSharePreemptionTimeout   = long.MaxValue;
         long             defaultMinSharePreemptionTimeout    = long.MaxValue;
         float            defaultFairSharePreemptionThreshold = 0.5f;
         SchedulingPolicy defaultSchedPolicy                  = SchedulingPolicy.DefaultPolicy;
         // Reservation global configuration knobs
         string planner                          = null;
         string reservationAgent                 = null;
         string reservationAdmissionPolicy       = null;
         QueuePlacementPolicy newPlacementPolicy = null;
         // Remember all queue names so we can display them on web UI, etc.
         // configuredQueues is segregated based on whether it is a leaf queue
         // or a parent queue. This information is used for creating queues
         // and also for making queue placement decisions(QueuePlacementRule.java).
         IDictionary <FSQueueType, ICollection <string> > configuredQueues = new Dictionary <FSQueueType
                                                                                             , ICollection <string> >();
         foreach (FSQueueType queueType in FSQueueType.Values())
         {
             configuredQueues[queueType] = new HashSet <string>();
         }
         // Read and parse the allocations file.
         DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
         docBuilderFactory.SetIgnoringComments(true);
         DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
         Document        doc     = builder.Parse(allocFile);
         Element         root    = doc.GetDocumentElement();
         if (!"allocations".Equals(root.GetTagName()))
         {
             throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>"
                                                        );
         }
         NodeList        elements               = root.GetChildNodes();
         IList <Element> queueElements          = new AList <Element>();
         Element         placementPolicyElement = null;
         for (int i = 0; i < elements.GetLength(); i++)
         {
             Node node = elements.Item(i);
             if (node is Element)
             {
                 Element element = (Element)node;
                 if ("queue".Equals(element.GetTagName()) || "pool".Equals(element.GetTagName()))
                 {
                     queueElements.AddItem(element);
                 }
                 else
                 {
                     if ("user".Equals(element.GetTagName()))
                     {
                         string   userName = element.GetAttribute("name");
                         NodeList fields   = element.GetChildNodes();
                         for (int j = 0; j < fields.GetLength(); j++)
                         {
                             Node fieldNode = fields.Item(j);
                             if (!(fieldNode is Element))
                             {
                                 continue;
                             }
                             Element field = (Element)fieldNode;
                             if ("maxRunningApps".Equals(field.GetTagName()))
                             {
                                 string text = ((Text)field.GetFirstChild()).GetData().Trim();
                                 int    val  = System.Convert.ToInt32(text);
                                 userMaxApps[userName] = val;
                             }
                         }
                     }
                     else
                     {
                         if ("userMaxAppsDefault".Equals(element.GetTagName()))
                         {
                             string text = ((Text)element.GetFirstChild()).GetData().Trim();
                             int    val  = System.Convert.ToInt32(text);
                             userMaxAppsDefault = val;
                         }
                         else
                         {
                             if ("defaultFairSharePreemptionTimeout".Equals(element.GetTagName()))
                             {
                                 string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                 long   val  = long.Parse(text) * 1000L;
                                 defaultFairSharePreemptionTimeout = val;
                             }
                             else
                             {
                                 if ("fairSharePreemptionTimeout".Equals(element.GetTagName()))
                                 {
                                     if (defaultFairSharePreemptionTimeout == long.MaxValue)
                                     {
                                         string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                         long   val  = long.Parse(text) * 1000L;
                                         defaultFairSharePreemptionTimeout = val;
                                     }
                                 }
                                 else
                                 {
                                     if ("defaultMinSharePreemptionTimeout".Equals(element.GetTagName()))
                                     {
                                         string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                         long   val  = long.Parse(text) * 1000L;
                                         defaultMinSharePreemptionTimeout = val;
                                     }
                                     else
                                     {
                                         if ("defaultFairSharePreemptionThreshold".Equals(element.GetTagName()))
                                         {
                                             string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                             float  val  = float.ParseFloat(text);
                                             val = Math.Max(Math.Min(val, 1.0f), 0.0f);
                                             defaultFairSharePreemptionThreshold = val;
                                         }
                                         else
                                         {
                                             if ("queueMaxAppsDefault".Equals(element.GetTagName()))
                                             {
                                                 string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                 int    val  = System.Convert.ToInt32(text);
                                                 queueMaxAppsDefault = val;
                                             }
                                             else
                                             {
                                                 if ("queueMaxAMShareDefault".Equals(element.GetTagName()))
                                                 {
                                                     string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                     float  val  = float.ParseFloat(text);
                                                     val = Math.Min(val, 1.0f);
                                                     queueMaxAMShareDefault = val;
                                                 }
                                                 else
                                                 {
                                                     if ("defaultQueueSchedulingPolicy".Equals(element.GetTagName()) || "defaultQueueSchedulingMode"
                                                         .Equals(element.GetTagName()))
                                                     {
                                                         string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                         defaultSchedPolicy = SchedulingPolicy.Parse(text);
                                                     }
                                                     else
                                                     {
                                                         if ("queuePlacementPolicy".Equals(element.GetTagName()))
                                                         {
                                                             placementPolicyElement = element;
                                                         }
                                                         else
                                                         {
                                                             if ("reservation-planner".Equals(element.GetTagName()))
                                                             {
                                                                 string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                                 planner = text;
                                                             }
                                                             else
                                                             {
                                                                 if ("reservation-agent".Equals(element.GetTagName()))
                                                                 {
                                                                     string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                                     reservationAgent = text;
                                                                 }
                                                                 else
                                                                 {
                                                                     if ("reservation-policy".Equals(element.GetTagName()))
                                                                     {
                                                                         string text = ((Text)element.GetFirstChild()).GetData().Trim();
                                                                         reservationAdmissionPolicy = text;
                                                                     }
                                                                     else
                                                                     {
                                                                         Log.Warn("Bad element in allocations file: " + element.GetTagName());
                                                                     }
                                                                 }
                                                             }
                                                         }
                                                     }
                                                 }
                                             }
                                         }
                                     }
                                 }
                             }
                         }
                     }
                 }
             }
         }
         // Load queue elements.  A root queue can either be included or omitted.  If
         // it's included, all other queues must be inside it.
         foreach (Element element_1 in queueElements)
         {
             string parent = "root";
             if (Sharpen.Runtime.EqualsIgnoreCase(element_1.GetAttribute("name"), "root"))
             {
                 if (queueElements.Count > 1)
                 {
                     throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it."
                                                                );
                 }
                 parent = null;
             }
             LoadQueue(parent, element_1, minQueueResources, maxQueueResources, queueMaxApps,
                       userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts
                       , fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues
                       , reservableQueues);
         }
         // Load placement policy and pass it configured queues
         Configuration conf = GetConfig();
         if (placementPolicyElement != null)
         {
             newPlacementPolicy = QueuePlacementPolicy.FromXml(placementPolicyElement, configuredQueues
                                                               , conf);
         }
         else
         {
             newPlacementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues
                                                                         );
         }
         // Set the min/fair share preemption timeout for the root queue
         if (!minSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
         {
             minSharePreemptionTimeouts[QueueManager.RootQueue] = defaultMinSharePreemptionTimeout;
         }
         if (!fairSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
         {
             fairSharePreemptionTimeouts[QueueManager.RootQueue] = defaultFairSharePreemptionTimeout;
         }
         // Set the fair share preemption threshold for the root queue
         if (!fairSharePreemptionThresholds.Contains(QueueManager.RootQueue))
         {
             fairSharePreemptionThresholds[QueueManager.RootQueue] = defaultFairSharePreemptionThreshold;
         }
         ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration
                                                                          ();
         if (planner != null)
         {
             globalReservationQueueConfig.SetPlanner(planner);
         }
         if (reservationAdmissionPolicy != null)
         {
             globalReservationQueueConfig.SetReservationAdmissionPolicy(reservationAdmissionPolicy
                                                                        );
         }
         if (reservationAgent != null)
         {
             globalReservationQueueConfig.SetReservationAgent(reservationAgent);
         }
         AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources
                                                                    , queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault,
                                                                    queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy,
                                                                    minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds
                                                                    , queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig,
                                                                    reservableQueues);
         lastSuccessfulReload    = clock.GetTime();
         lastReloadAttemptFailed = false;
         reloadListener.OnReload(info);
     }
 }
Exemplo n.º 13
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
 ///     "/>
 public abstract void SetPolicy(SchedulingPolicy policy);
Exemplo n.º 14
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
 ///     "/>
 protected internal virtual void ThrowPolicyDoesnotApplyException(SchedulingPolicy
                                                                  policy)
 {
     throw new AllocationConfigurationException("SchedulingPolicy " + policy + " does not apply to queue "
                                                + GetName());
 }
Exemplo n.º 15
0
        public virtual SchedulingPolicy GetSchedulingPolicy(string queueName)
        {
            SchedulingPolicy policy = schedulingPolicies[queueName];

            return((policy == null) ? defaultSchedulingPolicy : policy);
        }
Exemplo n.º 16
0
 /// <summary>
 /// Checks if the specified
 /// <see cref="SchedulingPolicy"/>
 /// can be used for a queue at
 /// the specified depth in the hierarchy
 /// </summary>
 /// <param name="policy">
 ///
 /// <see cref="SchedulingPolicy"/>
 /// we are checking the
 /// depth-applicability for
 /// </param>
 /// <param name="depth">queue's depth in the hierarchy</param>
 /// <returns>true if policy is applicable to passed depth, false otherwise</returns>
 public static bool IsApplicableTo(SchedulingPolicy policy, byte depth)
 {
     return(((policy.GetApplicableDepth() & depth) == depth) ? true : false);
 }