/// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
        ///     </summary>
        /// <remarks>
        /// Returns the fraction of CPU cores that should be used for YARN containers.
        /// The number is derived based on various configuration params such as
        /// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
        /// </remarks>
        /// <param name="conf">- Configuration object</param>
        /// <returns>Fraction of CPU cores to be used for YARN containers</returns>
        public static float GetContainersCores(Configuration conf)
        {
            ResourceCalculatorPlugin plugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
                                                  (null, conf);

            return(NodeManagerHardwareUtils.GetContainersCores(plugin, conf));
        }
示例#2
0
        public virtual void TestGetContainerCores()
        {
            YarnConfiguration conf = new YarnConfiguration();
            float             ret;
            int numProcessors = 4;
            ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
                                                                        >();

            Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 0);
            try
            {
                NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
                NUnit.Framework.Assert.Fail("getContainerCores should have thrown exception");
            }
            catch (ArgumentException)
            {
            }
            // expected
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 100);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(4, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 50);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(2, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 75);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(3, (int)ret);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 85);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(3.4, ret, 0.1);
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 110);
            ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            NUnit.Framework.Assert.AreEqual(4, (int)ret);
        }
示例#3
0
        internal virtual void Init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin
                                   )
        {
            InitConfig();
            // mount cgroups if requested
            if (cgroupMount && cgroupMountPath != null)
            {
                AList <string> cgroupKVs = new AList <string>();
                cgroupKVs.AddItem(ControllerCpu + "=" + cgroupMountPath + "/" + ControllerCpu);
                lce.MountCgroups(cgroupKVs, cgroupPrefix);
            }
            InitializeControllerPaths();
            // cap overall usage to the number of cores allocated to YARN
            yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
            int systemProcessors = plugin.GetNumProcessors();

            if (systemProcessors != (int)yarnProcessors)
            {
                Log.Info("YARN containers restricted to " + yarnProcessors + " cores");
                int[] limits = GetOverallLimits(yarnProcessors);
                UpdateCgroup(ControllerCpu, string.Empty, CpuPeriodUs, limits[0].ToString());
                UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, limits[1].ToString());
            }
            else
            {
                if (CpuLimitsExist())
                {
                    Log.Info("Removing CPU constraints for YARN containers.");
                    UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, (-1).ToString());
                }
            }
        }
        /// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
        ///     </summary>
        /// <remarks>
        /// Returns the fraction of CPU cores that should be used for YARN containers.
        /// The number is derived based on various configuration params such as
        /// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
        /// </remarks>
        /// <param name="plugin">- ResourceCalculatorPlugin object to determine hardware specs
        ///     </param>
        /// <param name="conf">- Configuration object</param>
        /// <returns>Fraction of CPU cores to be used for YARN containers</returns>
        public static float GetContainersCores(ResourceCalculatorPlugin plugin, Configuration
                                               conf)
        {
            int numProcessors     = plugin.GetNumProcessors();
            int nodeCpuPercentage = GetNodeCpuPercentage(conf);

            return((nodeCpuPercentage * numProcessors) / 100.0f);
        }
示例#5
0
        public virtual void TestInit()
        {
            LinuxContainerExecutor mockLCE = new TestCgroupsLCEResourcesHandler.MockLinuxContainerExecutor
                                                 ();

            TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler handler = new TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler
                                                                                         ();
            YarnConfiguration conf          = new YarnConfiguration();
            int numProcessors               = 4;
            ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
                                                                        >();

            Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
            handler.SetConf(conf);
            handler.InitConfig();
            // create mock cgroup
            FilePath cgroupMountDir = CreateMockCgroupMount(cgroupDir);
            // create mock mtab
            FilePath mockMtab = CreateMockMTab(cgroupDir);

            // setup our handler and call init()
            handler.SetMtabFile(mockMtab.GetAbsolutePath());
            // check values
            // in this case, we're using all cpu so the files
            // shouldn't exist(because init won't create them
            handler.Init(mockLCE, plugin);
            FilePath periodFile = new FilePath(cgroupMountDir, "cpu.cfs_period_us");
            FilePath quotaFile  = new FilePath(cgroupMountDir, "cpu.cfs_quota_us");

            NUnit.Framework.Assert.IsFalse(periodFile.Exists());
            NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
            // subset of cpu being used, files should be created
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 75);
            handler.limits[0] = 100 * 1000;
            handler.limits[1] = 1000 * 1000;
            handler.Init(mockLCE, plugin);
            int period = ReadIntFromFile(periodFile);
            int quota  = ReadIntFromFile(quotaFile);

            NUnit.Framework.Assert.AreEqual(100 * 1000, period);
            NUnit.Framework.Assert.AreEqual(1000 * 1000, quota);
            // set cpu back to 100, quota should be -1
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 100);
            handler.limits[0] = 100 * 1000;
            handler.limits[1] = 1000 * 1000;
            handler.Init(mockLCE, plugin);
            quota = ReadIntFromFile(quotaFile);
            NUnit.Framework.Assert.AreEqual(-1, quota);
            FileUtils.DeleteQuietly(cgroupDir);
        }
示例#6
0
        public virtual void TestRunCommandWithCpuAndMemoryResources()
        {
            // Windows only test
            Assume.AssumeTrue(Shell.Windows);
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.NmWindowsContainerCpuLimitEnabled, "true");
            conf.Set(YarnConfiguration.NmWindowsContainerMemoryLimitEnabled, "true");
            string[] command = containerExecutor.GetRunCommand("echo", "group1", null, null,
                                                               conf, Resource.NewInstance(1024, 1));
            float yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(ResourceCalculatorPlugin
                                                                               .GetResourceCalculatorPlugin(null, conf), conf);
            int cpuRate = Math.Min(10000, (int)((1 * 10000) / yarnProcessors));

            // Assert the cpu and memory limits are set correctly in the command
            string[] expected = new string[] { Shell.Winutils, "task", "create", "-m", "1024"
                                               , "-c", cpuRate.ToString(), "group1", "cmd /c " + "echo" };
            NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected, command));
        }
示例#7
0
        /// <exception cref="System.Exception"/>
        protected override void ServiceInit(Configuration conf)
        {
            this.monitoringInterval = conf.GetLong(YarnConfiguration.NmContainerMonIntervalMs
                                                   , YarnConfiguration.DefaultNmContainerMonIntervalMs);
            Type clazz = conf.GetClass <ResourceCalculatorPlugin>(YarnConfiguration.NmContainerMonResourceCalculator
                                                                  , null);

            this.resourceCalculatorPlugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
                                                (clazz, conf);
            Log.Info(" Using ResourceCalculatorPlugin : " + this.resourceCalculatorPlugin);
            processTreeClass = conf.GetClass <ResourceCalculatorProcessTree>(YarnConfiguration
                                                                             .NmContainerMonProcessTree, null);
            this.conf = conf;
            Log.Info(" Using ResourceCalculatorProcessTree : " + this.processTreeClass);
            this.containerMetricsEnabled = conf.GetBoolean(YarnConfiguration.NmContainerMetricsEnable
                                                           , YarnConfiguration.DefaultNmContainerMetricsEnable);
            this.containerMetricsPeriodMs = conf.GetLong(YarnConfiguration.NmContainerMetricsPeriodMs
                                                         , YarnConfiguration.DefaultNmContainerMetricsPeriodMs);
            this.containerMetricsUnregisterDelayMs = conf.GetLong(YarnConfiguration.NmContainerMetricsUnregisterDelayMs
                                                                  , YarnConfiguration.DefaultNmContainerMetricsUnregisterDelayMs);
            long configuredPMemForContainers = conf.GetLong(YarnConfiguration.NmPmemMb, YarnConfiguration
                                                            .DefaultNmPmemMb) * 1024 * 1024l;
            long configuredVCoresForContainers = conf.GetLong(YarnConfiguration.NmVcores, YarnConfiguration
                                                              .DefaultNmVcores);

            // Setting these irrespective of whether checks are enabled. Required in
            // the UI.
            // ///////// Physical memory configuration //////
            this.maxPmemAllottedForContainers   = configuredPMemForContainers;
            this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
            // ///////// Virtual memory configuration //////
            float vmemRatio = conf.GetFloat(YarnConfiguration.NmVmemPmemRatio, YarnConfiguration
                                            .DefaultNmVmemPmemRatio);

            Preconditions.CheckArgument(vmemRatio > 0.99f, YarnConfiguration.NmVmemPmemRatio
                                        + " should be at least 1.0");
            this.maxVmemAllottedForContainers = (long)(vmemRatio * configuredPMemForContainers
                                                       );
            pmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmPmemCheckEnabled, YarnConfiguration
                                               .DefaultNmPmemCheckEnabled);
            vmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmVmemCheckEnabled, YarnConfiguration
                                               .DefaultNmVmemCheckEnabled);
            Log.Info("Physical memory check enabled: " + pmemCheckEnabled);
            Log.Info("Virtual memory check enabled: " + vmemCheckEnabled);
            nodeCpuPercentageForYARN = NodeManagerHardwareUtils.GetNodeCpuPercentage(conf);
            if (pmemCheckEnabled)
            {
                // Logging if actual pmem cannot be determined.
                long totalPhysicalMemoryOnNM = UnknownMemoryLimit;
                if (this.resourceCalculatorPlugin != null)
                {
                    totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin.GetPhysicalMemorySize();
                    if (totalPhysicalMemoryOnNM <= 0)
                    {
                        Log.Warn("NodeManager's totalPmem could not be calculated. " + "Setting it to " +
                                 UnknownMemoryLimit);
                        totalPhysicalMemoryOnNM = UnknownMemoryLimit;
                    }
                }
                if (totalPhysicalMemoryOnNM != UnknownMemoryLimit && this.maxPmemAllottedForContainers
                    > totalPhysicalMemoryOnNM * 0.80f)
                {
                    Log.Warn("NodeManager configured with " + StringUtils.TraditionalBinaryPrefix.Long2String
                                 (maxPmemAllottedForContainers, string.Empty, 1) + " physical memory allocated to containers, which is more than "
                             + "80% of the total physical memory available (" + StringUtils.TraditionalBinaryPrefix
                             .Long2String(totalPhysicalMemoryOnNM, string.Empty, 1) + "). Thrashing might happen."
                             );
                }
            }
            base.ServiceInit(conf);
        }
示例#8
0
        public virtual void TestContainerLimits()
        {
            LinuxContainerExecutor mockLCE = new TestCgroupsLCEResourcesHandler.MockLinuxContainerExecutor
                                                 ();

            TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler handler = new TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler
                                                                                         ();
            handler.generateLimitsMode = true;
            YarnConfiguration conf          = new YarnConfiguration();
            int numProcessors               = 4;
            ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
                                                                        >();

            Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
            handler.SetConf(conf);
            handler.InitConfig();
            // create mock cgroup
            FilePath cgroupMountDir = CreateMockCgroupMount(cgroupDir);
            // create mock mtab
            FilePath mockMtab = CreateMockMTab(cgroupDir);

            // setup our handler and call init()
            handler.SetMtabFile(mockMtab.GetAbsolutePath());
            handler.Init(mockLCE, plugin);
            // check values
            // default case - files shouldn't exist, strict mode off by default
            ContainerId id = ContainerId.FromString("container_1_1_1_1");

            handler.PreExecute(id, Resource.NewInstance(1024, 1));
            FilePath containerDir = new FilePath(cgroupMountDir, id.ToString());

            NUnit.Framework.Assert.IsTrue(containerDir.Exists());
            NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
            FilePath periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
            FilePath quotaFile  = new FilePath(containerDir, "cpu.cfs_quota_us");

            NUnit.Framework.Assert.IsFalse(periodFile.Exists());
            NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
            // no files created because we're using all cpu
            FileUtils.DeleteQuietly(containerDir);
            conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
                            );
            handler.InitConfig();
            handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
                                                        ));
            NUnit.Framework.Assert.IsTrue(containerDir.Exists());
            NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
            periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
            quotaFile  = new FilePath(containerDir, "cpu.cfs_quota_us");
            NUnit.Framework.Assert.IsFalse(periodFile.Exists());
            NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
            // 50% of CPU
            FileUtils.DeleteQuietly(containerDir);
            conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
                            );
            handler.InitConfig();
            handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
                                                        / 2));
            NUnit.Framework.Assert.IsTrue(containerDir.Exists());
            NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
            periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
            quotaFile  = new FilePath(containerDir, "cpu.cfs_quota_us");
            NUnit.Framework.Assert.IsTrue(periodFile.Exists());
            NUnit.Framework.Assert.IsTrue(quotaFile.Exists());
            NUnit.Framework.Assert.AreEqual(500 * 1000, ReadIntFromFile(periodFile));
            NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(quotaFile));
            // CGroups set to 50% of CPU, container set to 50% of YARN CPU
            FileUtils.DeleteQuietly(containerDir);
            conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
                            );
            conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 50);
            handler.InitConfig();
            handler.Init(mockLCE, plugin);
            handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
                                                        / 2));
            NUnit.Framework.Assert.IsTrue(containerDir.Exists());
            NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
            periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
            quotaFile  = new FilePath(containerDir, "cpu.cfs_quota_us");
            NUnit.Framework.Assert.IsTrue(periodFile.Exists());
            NUnit.Framework.Assert.IsTrue(quotaFile.Exists());
            NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(periodFile));
            NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(quotaFile));
            FileUtils.DeleteQuietly(cgroupDir);
        }
示例#9
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Init(LinuxContainerExecutor lce)
 {
     this.Init(lce, ResourceCalculatorPlugin.GetResourceCalculatorPlugin(null, conf));
 }