Example #1
0
        public virtual void TestGetContainerCores()
        {
            YarnConfiguration conf = new YarnConfiguration();
            float             ret;
            int numProcessors = 4;
            ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
                                                                        >();

            Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 0);
            try
            {
                NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
                NUnit.Framework.Assert.Fail("getContainerCores should have thrown exception");
            }
            catch (ArgumentException)
            {
            }
            // expected
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 100);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(4, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 50);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(2, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 75);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(3, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 85);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(3.4, ret, 0.1);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 110);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(4, (int)ret);
        }
        /// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
        ///     </summary>
        /// <remarks>
        /// Returns the fraction of CPU cores that should be used for YARN containers.
        /// The number is derived based on various configuration params such as
        /// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
        /// </remarks>
        /// <param name="conf">- Configuration object</param>
        /// <returns>Fraction of CPU cores to be used for YARN containers</returns>
        public static float GetContainersCores(Configuration conf)
        {
            ResourceCalculatorPlugin plugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
                                                  (null, conf);

            return(NodeManagerHardwareUtils.GetContainersCores(plugin, conf));
        }
Example #3
0
        internal virtual void Init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin
                                   )
        {
            InitConfig();
            // mount cgroups if requested
            if (cgroupMount && cgroupMountPath != null)
            {
                AList <string> cgroupKVs = new AList <string>();
                cgroupKVs.AddItem(ControllerCpu + "=" + cgroupMountPath + "/" + ControllerCpu);
                lce.MountCgroups(cgroupKVs, cgroupPrefix);
            }
            InitializeControllerPaths();
            // cap overall usage to the number of cores allocated to YARN
            yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            int systemProcessors = plugin.GetNumProcessors();

            if (systemProcessors != (int)yarnProcessors)
            {
                Log.Info("YARN containers restricted to " + yarnProcessors + " cores");
                int[] limits = GetOverallLimits(yarnProcessors);
                UpdateCgroup(ControllerCpu, string.Empty, CpuPeriodUs, limits[0].ToString());
                UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, limits[1].ToString());
            }
            else
            {
                if (CpuLimitsExist())
                {
                    Log.Info("Removing CPU constraints for YARN containers.");
                    UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, (-1).ToString());
                }
            }
        }