Пример #1
0
 /// <exception cref="System.IO.IOException"/>
 public override void Reinitialize(CSQueue newlyParsedQueue, Resource clusterResource
                                   )
 {
     lock (this)
     {
         // Sanity check
         if (!(newlyParsedQueue is Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue
               ) || !newlyParsedQueue.GetQueuePath().Equals(GetQueuePath()))
         {
             throw new IOException("Trying to reinitialize " + GetQueuePath() + " from " + newlyParsedQueue
                                   .GetQueuePath());
         }
         Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue newlyParsedParentQueue
             = (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.PlanQueue)newlyParsedQueue;
         if (newlyParsedParentQueue.GetChildQueues().Count > 0)
         {
             throw new IOException("Reservable Queue should not have sub-queues in the" + "configuration"
                                   );
         }
         // Set new configs
         SetupQueueConfigs(clusterResource);
         UpdateQuotas(newlyParsedParentQueue.userLimit, newlyParsedParentQueue.userLimitFactor
                      , newlyParsedParentQueue.maxAppsForReservation, newlyParsedParentQueue.maxAppsPerUserForReservation
                      );
         // run reinitialize on each existing queue, to trigger absolute cap
         // recomputations
         foreach (CSQueue res in this.GetChildQueues())
         {
             res.Reinitialize(res, clusterResource);
         }
         showReservationsAsQueues = newlyParsedParentQueue.showReservationsAsQueues;
     }
 }
Пример #2
0
        public virtual void TestOffSwitchSchedulingMultiLevelQueues()
        {
            // Setup queue configs
            SetupMultiLevelQueues(csConf);
            //B3
            IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
            CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
                                                        .Root, queues, queues, TestUtils.spyHook);
            // Setup some nodes
            int memoryPerNode        = 10;
            int coresPerNode         = 10;
            int numNodes             = 2;
            FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
                                                             * Gb);
            FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
                                                             * Gb);

            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
                                                                              (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
            Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
            // Start testing
            LeafQueue b3 = (LeafQueue)queues[B3];
            LeafQueue b2 = (LeafQueue)queues[B2];

            // Simulate B3 returning a container on node_0
            StubQueueAllocation(b2, clusterResource, node_0, 0 * Gb, NodeType.OffSwitch);
            StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            VerifyQueueMetrics(b2, 0 * Gb, clusterResource);
            VerifyQueueMetrics(b3, 1 * Gb, clusterResource);
            // Now, B2 should get the scheduling opportunity since B2=0G/2G, B3=1G/7G
            // also, B3 gets a scheduling opportunity since B2 allocates RACK_LOCAL
            StubQueueAllocation(b2, clusterResource, node_1, 1 * Gb, NodeType.RackLocal);
            StubQueueAllocation(b3, clusterResource, node_1, 1 * Gb, NodeType.OffSwitch);
            root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
                                                                              ));
            InOrder allocationOrder = Org.Mockito.Mockito.InOrder(b2, b3);

            allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                        <FiCaSchedulerNode>(), AnyResourceLimits());
            allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                        <FiCaSchedulerNode>(), AnyResourceLimits());
            VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
            VerifyQueueMetrics(b3, 2 * Gb, clusterResource);
            // Now, B3 should get the scheduling opportunity
            // since B2 has 1/2G while B3 has 2/7G,
            // However, since B3 returns off-switch, B2 won't get an opportunity
            StubQueueAllocation(b2, clusterResource, node_0, 1 * Gb, NodeType.NodeLocal);
            StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            allocationOrder = Org.Mockito.Mockito.InOrder(b3, b2);
            allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                        <FiCaSchedulerNode>(), AnyResourceLimits());
            allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                        <FiCaSchedulerNode>(), AnyResourceLimits());
            VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
            VerifyQueueMetrics(b3, 3 * Gb, clusterResource);
        }
Пример #3
0
        /// <exception cref="System.IO.IOException"/>
        public PlanQueue(CapacitySchedulerContext cs, string queueName, CSQueue parent, CSQueue
                         old)
            : base(cs, queueName, parent, old)
        {
            this.schedulerContext = cs;
            // Set the reservation queue attributes for the Plan
            CapacitySchedulerConfiguration conf = cs.GetConfiguration();
            string queuePath             = base.GetQueuePath();
            int    maxAppsForReservation = conf.GetMaximumApplicationsPerQueue(queuePath);

            showReservationsAsQueues = conf.GetShowReservationAsQueues(queuePath);
            if (maxAppsForReservation < 0)
            {
                maxAppsForReservation = (int)(CapacitySchedulerConfiguration.DefaultMaximumSystemApplicatiions
                                              * base.GetAbsoluteCapacity());
            }
            int   userLimit       = conf.GetUserLimit(queuePath);
            float userLimitFactor = conf.GetUserLimitFactor(queuePath);
            int   maxAppsPerUserForReservation = (int)(maxAppsForReservation * (userLimit / 100.0f
                                                                                ) * userLimitFactor);

            UpdateQuotas(userLimit, userLimitFactor, maxAppsForReservation, maxAppsPerUserForReservation
                         );
            StringBuilder queueInfo = new StringBuilder();

            queueInfo.Append("Created Plan Queue: ").Append(queueName).Append("\nwith capacity: ["
                                                                              ).Append(base.GetCapacity()).Append("]\nwith max capacity: [").Append(base.GetMaximumCapacity
                                                                                                                                                        ()).Append("\nwith max reservation apps: [").Append(maxAppsForReservation).Append
                ("]\nwith max reservation apps per user: [").Append(maxAppsPerUserForReservation
                                                                    ).Append("]\nwith user limit: [").Append(userLimit).Append("]\nwith user limit factor: ["
                                                                                                                               ).Append(userLimitFactor).Append("].");
            Log.Info(queueInfo.ToString());
        }
Пример #4
0
        public virtual void TestNestedQueueParsingShouldTrimSpaces()
        {
            CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();

            SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
            YarnConfiguration conf = new YarnConfiguration(csConf);
            CapacityScheduler capacityScheduler = new CapacityScheduler();

            capacityScheduler.SetConf(conf);
            capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
            capacityScheduler.Init(conf);
            capacityScheduler.Start();
            capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
            CSQueue a = capacityScheduler.GetQueue("a");

            NUnit.Framework.Assert.IsNotNull(a);
            NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
            CSQueue c = capacityScheduler.GetQueue("c");

            NUnit.Framework.Assert.IsNotNull(c);
            NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
            CSQueue a1 = capacityScheduler.GetQueue("a1");

            NUnit.Framework.Assert.IsNotNull(a1);
            NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta);
            CSQueue a2 = capacityScheduler.GetQueue("a2");

            NUnit.Framework.Assert.IsNotNull(a2);
            NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta);
        }
Пример #5
0
        public virtual void TestQueueParsing()
        {
            CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();

            SetupQueueConfiguration(csConf);
            YarnConfiguration conf = new YarnConfiguration(csConf);
            CapacityScheduler capacityScheduler = new CapacityScheduler();

            capacityScheduler.SetConf(conf);
            capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
            capacityScheduler.Init(conf);
            capacityScheduler.Start();
            capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
            CSQueue a = capacityScheduler.GetQueue("a");

            NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
            CSQueue b1 = capacityScheduler.GetQueue("b1");

            NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
                                                (), Delta);
            CSQueue c12 = capacityScheduler.GetQueue("c12");

            NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
                                            );
            NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
                                                ), Delta);
            ServiceOperations.StopQuietly(capacityScheduler);
        }
Пример #6
0
 protected override void Render(HtmlBlock.Block html)
 {
     html.(typeof(MetricsOverviewTable));
     Hamlet.UL <Hamlet.DIV <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > ul
         = html.Div("#cs-wrapper.ui-widget").Div(".ui-widget-header.ui-corner-top").("Application Queues"
                                                                                     ).().Div("#cs.ui-widget-content.ui-corner-bottom").Ul();
     if (cs == null)
     {
         ul.Li().A(Q).$style(Width(QMaxWidth)).Span().$style(QEnd).("100% ").().Span(".q",
                                                                                     "default").().();
     }
     else
     {
         CSQueue root = cs.GetRootQueue();
         CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root);
         csqinfo.csinfo = sinfo;
         csqinfo.qinfo  = null;
         float used = sinfo.GetUsedCapacity() / 100;
         ul.Li().$style("margin-bottom: 1em").Span().$style("font-weight: bold").("Legend:"
                                                                                  ).().Span().$class("qlegend ui-corner-all").$style(QGiven).("Capacity").().Span(
             ).$class("qlegend ui-corner-all").$style(QUnder).("Used").().Span().$class("qlegend ui-corner-all"
                                                                                        ).$style(QOver).("Used (over capacity)").().Span().$class("qlegend ui-corner-all ui-state-default"
                                                                                                                                                  ).("Max Capacity").().().Li().A(Q).$style(Width(QMaxWidth)).Span().$style(StringHelper.Join
                                                                                                                                                                                                                                (Width(used), ";left:0%;", used > 1 ? QOver : QUnder)).(".").().Span(".q", "root"
                                                                                                                                                                                                                                                                                                     ).().Span().$class("qstats").$style(Left(QStatsPos)).(StringHelper.Join(Percent(
                                                                                                                                                                                                                                                                                                                                                                                 used), " used")).().(typeof(CapacitySchedulerPage.QueueBlock)).();
     }
     ul.().().Script().$type("text/javascript").("$('#cs').hide();").().().(typeof(RMAppsBlock
                                                                                   ));
 }
Пример #7
0
        internal CapacitySchedulerQueueInfo(CSQueue q)
        {
            queuePath    = q.GetQueuePath();
            capacity     = q.GetCapacity() * 100;
            usedCapacity = q.GetUsedCapacity() * 100;
            maxCapacity  = q.GetMaximumCapacity();
            if (maxCapacity < Epsilon || maxCapacity > 1f)
            {
                maxCapacity = 1f;
            }
            maxCapacity         *= 100;
            absoluteCapacity     = Cap(q.GetAbsoluteCapacity(), 0f, 1f) * 100;
            absoluteMaxCapacity  = Cap(q.GetAbsoluteMaximumCapacity(), 0f, 1f) * 100;
            absoluteUsedCapacity = Cap(q.GetAbsoluteUsedCapacity(), 0f, 1f) * 100;
            numApplications      = q.GetNumApplications();
            queueName            = q.GetQueueName();
            state         = q.GetState();
            resourcesUsed = new ResourceInfo(q.GetUsedResources());
            if (q is PlanQueue && !((PlanQueue)q).ShowReservationsAsQueues())
            {
                hideReservationQueues = true;
            }
            // add labels
            ICollection <string> labelSet = q.GetAccessibleNodeLabels();

            if (labelSet != null)
            {
                Sharpen.Collections.AddAll(nodeLabels, labelSet);
                nodeLabels.Sort();
            }
        }
Пример #8
0
        protected internal override void AssertReservationQueueDoesNotExist(ReservationId
                                                                            r2)
        {
            CSQueue q2 = cs.GetQueue(r2.ToString());

            NUnit.Framework.Assert.IsNull(q2);
        }
Пример #9
0
        protected internal override int GetNumberOfApplications(Queue queue)
        {
            CSQueue csQueue = (CSQueue)queue;
            int     numberOfApplications = csQueue.GetNumApplications();

            return(numberOfApplications);
        }
Пример #10
0
        public static void UpdateQueueStatistics(ResourceCalculator calculator, CSQueue childQueue
                                                 , CSQueue parentQueue, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource
                                                 , Org.Apache.Hadoop.Yarn.Api.Records.Resource minimumAllocation)
        {
            Org.Apache.Hadoop.Yarn.Api.Records.Resource queueLimit    = Resources.None();
            Org.Apache.Hadoop.Yarn.Api.Records.Resource usedResources = childQueue.GetUsedResources
                                                                            ();
            float absoluteUsedCapacity = 0.0f;
            float usedCapacity         = 0.0f;

            if (Resources.GreaterThan(calculator, clusterResource, clusterResource, Resources
                                      .None()))
            {
                queueLimit = Resources.Multiply(clusterResource, childQueue.GetAbsoluteCapacity()
                                                );
                absoluteUsedCapacity = Resources.Divide(calculator, clusterResource, usedResources
                                                        , clusterResource);
                usedCapacity = Resources.Equals(queueLimit, Resources.None()) ? 0 : Resources.Divide
                                   (calculator, clusterResource, usedResources, queueLimit);
            }
            childQueue.SetUsedCapacity(usedCapacity);
            childQueue.SetAbsoluteUsedCapacity(absoluteUsedCapacity);
            Org.Apache.Hadoop.Yarn.Api.Records.Resource available = Resources.Subtract(queueLimit
                                                                                       , usedResources);
            childQueue.GetMetrics().SetAvailableResourcesToQueue(Resources.Max(calculator, clusterResource
                                                                               , available, Resources.None()));
        }
Пример #11
0
        /// <summary>
        /// The specified queue is preemptable if system-wide preemption is turned on
        /// unless any queue in the <em>qPath</em> hierarchy has explicitly turned
        /// preemption off.
        /// </summary>
        /// <remarks>
        /// The specified queue is preemptable if system-wide preemption is turned on
        /// unless any queue in the <em>qPath</em> hierarchy has explicitly turned
        /// preemption off.
        /// NOTE: Preemptability is inherited from a queue's parent.
        /// </remarks>
        /// <returns>true if queue has preemption disabled, false otherwise</returns>
        private bool IsQueueHierarchyPreemptionDisabled(CSQueue q)
        {
            CapacitySchedulerConfiguration csConf = csContext.GetConfiguration();
            bool systemWidePreemption             = csConf.GetBoolean(YarnConfiguration.RmSchedulerEnableMonitors
                                                                      , YarnConfiguration.DefaultRmSchedulerEnableMonitors);
            CSQueue parentQ = q.GetParent();

            // If the system-wide preemption switch is turned off, all of the queues in
            // the qPath hierarchy have preemption disabled, so return true.
            if (!systemWidePreemption)
            {
                return(true);
            }
            // If q is the root queue and the system-wide preemption switch is turned
            // on, then q does not have preemption disabled (default=false, below)
            // unless the preemption_disabled property is explicitly set.
            if (parentQ == null)
            {
                return(csConf.GetPreemptionDisabled(q.GetQueuePath(), false));
            }
            // If this is not the root queue, inherit the default value for the
            // preemption_disabled property from the parent. Preemptability will be
            // inherited from the parent's hierarchy unless explicitly overridden at
            // this level.
            return(csConf.GetPreemptionDisabled(q.GetQueuePath(), parentQ.GetPreemptionDisabled
                                                    ()));
        }
Пример #12
0
 public virtual void SetParent(CSQueue newParentQueue)
 {
     lock (this)
     {
         this.parent = (ParentQueue)newParentQueue;
     }
 }
Пример #13
0
 public CSRate(Employee emp)
 {
     this.csq      = CSQueue.getInstance();
     this.connect  = ConnectDatabase.getInstance();
     this.employee = emp;
     InitializeComponent();
     employeename.Content = emp.name;
 }
Пример #14
0
 private void VerifyQueueMetrics(CSQueue queue, int expectedMemory, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                 clusterResource)
 {
     NUnit.Framework.Assert.AreEqual(ComputeQueueAbsoluteUsedCapacity(queue, expectedMemory
                                                                      , clusterResource), queue.GetAbsoluteUsedCapacity(), Delta);
     NUnit.Framework.Assert.AreEqual(ComputeQueueUsedCapacity(queue, expectedMemory, clusterResource
                                                              ), queue.GetUsedCapacity(), Delta);
 }
Пример #15
0
        public static float ComputeAbsoluteMaximumCapacity(float maximumCapacity, CSQueue
                                                           parent)
        {
            float parentAbsMaxCapacity = (parent == null) ? 1.0f : parent.GetAbsoluteMaximumCapacity
                                             ();

            return(parentAbsMaxCapacity * maximumCapacity);
        }
Пример #16
0
 private void StubQueueAllocation(CSQueue queue, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                  clusterResource, FiCaSchedulerNode node, int allocation, NodeType type)
 {
     // Simulate the queue allocation
     Org.Mockito.Mockito.DoAnswer(new _Answer_137(this, queue, allocation, node, clusterResource
                                                  , type)).When(queue).AssignContainers(Matchers.Eq(clusterResource), Matchers.Eq(
                                                                                            node), Matchers.Any <ResourceLimits>());
 }
Пример #17
0
        protected internal override void AssertReservationQueueExists(ReservationId r2, double
                                                                      expectedCapacity, double expectedMaxCapacity)
        {
            CSQueue q = cs.GetQueue(r2.ToString());

            NUnit.Framework.Assert.IsNotNull(q);
            NUnit.Framework.Assert.AreEqual(expectedCapacity, q.GetCapacity(), 0.01);
            NUnit.Framework.Assert.AreEqual(expectedMaxCapacity, q.GetMaximumCapacity(), 1.0);
        }
Пример #18
0
        private void CheckUsedResource(MockRM rm, string queueName, int memory, string label
                                       )
        {
            CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
            CSQueue           queue     = scheduler.GetQueue(queueName);

            NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label
                                                                                          ).GetMemory());
        }
Пример #19
0
 public QMShowQueue()
 {
     tq           = TellerQueue.getInstance();
     csq          = CSQueue.getInstance();
     this.connect = ConnectDatabase.getInstance();
     InitializeComponent();
     tellerqueue.Content = "";
     csqueue.Content     = "";
     init();
 }
        protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetPlanQueueCapacity
            (string planQueueName)
        {
            Org.Apache.Hadoop.Yarn.Api.Records.Resource minAllocation = GetMinAllocation();
            ResourceCalculator rescCalc  = GetResourceCalculator();
            CSQueue            planQueue = capScheduler.GetQueue(planQueueName);

            return(rescCalc.MultiplyAndNormalizeDown(capScheduler.GetClusterResource(), planQueue
                                                     .GetAbsoluteCapacity(), minAllocation));
        }
        protected internal override Queue GetPlanQueue(string planQueueName)
        {
            CSQueue queue = cs.GetQueue(planQueueName);

            if (!(queue is PlanQueue))
            {
                Log.Error("The Plan is not an PlanQueue!");
                return(null);
            }
            return(queue);
        }
Пример #22
0
 public _Answer_137(TestParentQueue _enclosing, CSQueue queue, int allocation, FiCaSchedulerNode
                    node, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource, NodeType type
                    )
 {
     this._enclosing      = _enclosing;
     this.queue           = queue;
     this.allocation      = allocation;
     this.node            = node;
     this.clusterResource = clusterResource;
     this.type            = type;
 }
Пример #23
0
 private void StubQueueAllocation(CSQueue queue, Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                  clusterResource, FiCaSchedulerNode node, int allocation, NodeType type)
 {
     // Simulate the queue allocation
     Org.Mockito.Mockito.DoAnswer(new _Answer_124(this, queue, allocation, node, clusterResource
                                                  , type)).When(queue).AssignContainers(Matchers.Eq(clusterResource), Matchers.Eq(
                                                                                            node), Matchers.Any <ResourceLimits>());
     // Next call - nothing
     // Mock the node's resource availability
     Org.Mockito.Mockito.DoNothing().When(node).ReleaseContainer(Matchers.Any <Container
                                                                               >());
 }
Пример #24
0
        public CSQRCode(Employee emp, Customer cust)
        {
            this.csq      = CSQueue.getInstance();
            this.employee = emp;
            this.customer = cust;
            InitializeComponent();
            QRCodeGenerator qrGenerator  = new QRCodeGenerator();
            QRCodeData      qrCodeData   = qrGenerator.CreateQrCode(csq.uniquequeue, QRCodeGenerator.ECCLevel.H);
            XamlQRCode      qrCode       = new XamlQRCode(qrCodeData);
            DrawingImage    qrCodeAsXaml = qrCode.GetGraphic(20);

            qrcode.Source = qrCodeAsXaml;
        }
        protected internal override Org.Apache.Hadoop.Yarn.Api.Records.Resource GetReservationQueueResourceIfExists
            (Plan plan, ReservationId reservationId)
        {
            CSQueue resQueue = cs.GetQueue(reservationId.ToString());

            Org.Apache.Hadoop.Yarn.Api.Records.Resource reservationResource = null;
            if (resQueue != null)
            {
                reservationResource = Resources.Multiply(cs.GetClusterResource(), resQueue.GetAbsoluteCapacity
                                                             ());
            }
            return(reservationResource);
        }
Пример #26
0
 /// <exception cref="System.IO.IOException"/>
 public override void Reinitialize(CSQueue newlyParsedQueue, Resource clusterResource
                                   )
 {
     lock (this)
     {
         // Sanity check
         if (!(newlyParsedQueue is Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue
               ) || !newlyParsedQueue.GetQueuePath().Equals(GetQueuePath()))
         {
             throw new IOException("Trying to reinitialize " + GetQueuePath() + " from " + newlyParsedQueue
                                   .GetQueuePath());
         }
         Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue newlyParsedParentQueue
             = (Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Capacity.ParentQueue
                )newlyParsedQueue;
         // Set new configs
         SetupQueueConfigs(clusterResource);
         // Re-configure existing child queues and add new ones
         // The CS has already checked to ensure all existing child queues are present!
         IDictionary <string, CSQueue> currentChildQueues = GetQueues(childQueues);
         IDictionary <string, CSQueue> newChildQueues     = GetQueues(newlyParsedParentQueue.childQueues
                                                                      );
         foreach (KeyValuePair <string, CSQueue> e in newChildQueues)
         {
             string  newChildQueueName = e.Key;
             CSQueue newChildQueue     = e.Value;
             CSQueue childQueue        = currentChildQueues[newChildQueueName];
             // Check if the child-queue already exists
             if (childQueue != null)
             {
                 // Re-init existing child queues
                 childQueue.Reinitialize(newChildQueue, clusterResource);
                 Log.Info(GetQueueName() + ": re-configured queue: " + childQueue);
             }
             else
             {
                 // New child queue, do not re-init
                 // Set parent to 'this'
                 newChildQueue.SetParent(this);
                 // Save in list of current child queues
                 currentChildQueues[newChildQueueName] = newChildQueue;
                 Log.Info(GetQueueName() + ": added new child queue: " + newChildQueue);
             }
         }
         // Re-sort all queues
         childQueues.Clear();
         Sharpen.Collections.AddAll(childQueues, currentChildQueues.Values);
     }
 }
Пример #27
0
        public CapacitySchedulerInfo(CSQueue parent)
        {
            // JAXB needs this
            this.queueName    = parent.GetQueueName();
            this.usedCapacity = parent.GetUsedCapacity() * 100;
            this.capacity     = parent.GetCapacity() * 100;
            float max = parent.GetMaximumCapacity();

            if (max < Epsilon || max > 1f)
            {
                max = 1f;
            }
            this.maxCapacity = max * 100;
            queues           = GetQueues(parent);
        }
Пример #28
0
        public virtual void TestQueueParsingWithUnusedLabels()
        {
            ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");

            // Initialize a cluster with labels, but doesn't use them, reinitialize
            // shouldn't fail
            nodeLabelManager.AddToCluserNodeLabels(labels);
            CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();

            SetupQueueConfiguration(csConf);
            csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
            YarnConfiguration conf = new YarnConfiguration(csConf);
            CapacityScheduler capacityScheduler = new CapacityScheduler();

            capacityScheduler.SetConf(conf);
            RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
                                                        RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
                                                        ClientToAMTokenSecretManagerInRM(), null);

            rmContext.SetNodeLabelManager(nodeLabelManager);
            capacityScheduler.SetRMContext(rmContext);
            capacityScheduler.Init(conf);
            capacityScheduler.Start();
            capacityScheduler.Reinitialize(conf, rmContext);
            // check root queue's capacity by label -- they should be all zero
            CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);

            NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
                                            Delta);
            NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
                                            Delta);
            CSQueue a = capacityScheduler.GetQueue("a");

            NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
            CSQueue b1 = capacityScheduler.GetQueue("b1");

            NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
            NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
                                                (), Delta);
            CSQueue c12 = capacityScheduler.GetQueue("c12");

            NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
                                            );
            NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
                                                ), Delta);
            capacityScheduler.Stop();
        }
Пример #29
0
 /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.SchedulerDynamicEditException
 ///     "/>
 internal virtual void AddChildQueue(CSQueue newQueue)
 {
     lock (this)
     {
         if (newQueue.GetCapacity() > 0)
         {
             throw new SchedulerDynamicEditException("Queue " + newQueue + " being added has non zero capacity."
                                                     );
         }
         bool added = this.childQueues.AddItem(newQueue);
         if (Log.IsDebugEnabled())
         {
             Log.Debug("updateChildQueues (action: add queue): " + added + " " + GetChildQueuesToPrint
                           ());
         }
     }
 }
Пример #30
0
 public override void CompletedContainer(Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                         clusterResource, FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer
                                         rmContainer, ContainerStatus containerStatus, RMContainerEventType @event, CSQueue
                                         completedChildQueue, bool sortQueues)
 {
     if (application != null)
     {
         // Careful! Locking order is important!
         // Book keeping
         lock (this)
         {
             base.ReleaseResource(clusterResource, rmContainer.GetContainer().GetResource(), node
                                  .GetLabels());
             Log.Info("completedContainer" + " queue=" + GetQueueName() + " usedCapacity=" + GetUsedCapacity
                          () + " absoluteUsedCapacity=" + GetAbsoluteUsedCapacity() + " used=" + queueUsage
                      .GetUsed() + " cluster=" + clusterResource);
             // Note that this is using an iterator on the childQueues so this can't
             // be called if already within an iterator for the childQueues. Like
             // from assignContainersToChildQueues.
             if (sortQueues)
             {
                 // reinsert the updated queue
                 for (IEnumerator <CSQueue> iter = childQueues.GetEnumerator(); iter.HasNext();)
                 {
                     CSQueue csqueue = iter.Next();
                     if (csqueue.Equals(completedChildQueue))
                     {
                         iter.Remove();
                         Log.Info("Re-sorting completed queue: " + csqueue.GetQueuePath() + " stats: " + csqueue
                                  );
                         childQueues.AddItem(csqueue);
                         break;
                     }
                 }
             }
         }
         // Inform the parent
         if (parent != null)
         {
             // complete my parent
             parent.CompletedContainer(clusterResource, application, node, rmContainer, null,
                                       @event, this, sortQueues);
         }
     }
 }