/// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
/// </summary>
/// <remarks>
/// Returns the fraction of CPU cores that should be used for YARN containers.
/// The number is derived based on various configuration params such as
/// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
/// </remarks>
/// <param name="conf">- Configuration object</param>
/// <returns>Fraction of CPU cores to be used for YARN containers</returns>
public static float GetContainersCores(Configuration conf)
{
ResourceCalculatorPlugin plugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
(null, conf);
return(NodeManagerHardwareUtils.GetContainersCores(plugin, conf));
}
public virtual void TestGetContainerCores()
{
YarnConfiguration conf = new YarnConfiguration();
float ret;
int numProcessors = 4;
ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
>();
Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 0);
try
{
NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.Fail("getContainerCores should have thrown exception");
}
catch (ArgumentException)
{
}
// expected
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 100);
ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.AreEqual(4, (int)ret);
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 50);
ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.AreEqual(2, (int)ret);
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 75);
ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.AreEqual(3, (int)ret);
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 85);
ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.AreEqual(3.4, ret, 0.1);
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 110);
ret = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
NUnit.Framework.Assert.AreEqual(4, (int)ret);
}
internal virtual void Init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin
)
{
InitConfig();
// mount cgroups if requested
if (cgroupMount && cgroupMountPath != null)
{
AList <string> cgroupKVs = new AList <string>();
cgroupKVs.AddItem(ControllerCpu + "=" + cgroupMountPath + "/" + ControllerCpu);
lce.MountCgroups(cgroupKVs, cgroupPrefix);
}
InitializeControllerPaths();
// cap overall usage to the number of cores allocated to YARN
yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(plugin, conf);
int systemProcessors = plugin.GetNumProcessors();
if (systemProcessors != (int)yarnProcessors)
{
Log.Info("YARN containers restricted to " + yarnProcessors + " cores");
int[] limits = GetOverallLimits(yarnProcessors);
UpdateCgroup(ControllerCpu, string.Empty, CpuPeriodUs, limits[0].ToString());
UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, limits[1].ToString());
}
else
{
if (CpuLimitsExist())
{
Log.Info("Removing CPU constraints for YARN containers.");
UpdateCgroup(ControllerCpu, string.Empty, CpuQuotaUs, (-1).ToString());
}
}
}
/// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
/// </summary>
/// <remarks>
/// Returns the fraction of CPU cores that should be used for YARN containers.
/// The number is derived based on various configuration params such as
/// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
/// </remarks>
/// <param name="plugin">- ResourceCalculatorPlugin object to determine hardware specs
/// </param>
/// <param name="conf">- Configuration object</param>
/// <returns>Fraction of CPU cores to be used for YARN containers</returns>
public static float GetContainersCores(ResourceCalculatorPlugin plugin, Configuration
conf)
{
int numProcessors = plugin.GetNumProcessors();
int nodeCpuPercentage = GetNodeCpuPercentage(conf);
return((nodeCpuPercentage * numProcessors) / 100.0f);
}
public virtual void TestInit()
{
LinuxContainerExecutor mockLCE = new TestCgroupsLCEResourcesHandler.MockLinuxContainerExecutor
();
TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler handler = new TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler
();
YarnConfiguration conf = new YarnConfiguration();
int numProcessors = 4;
ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
>();
Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
handler.SetConf(conf);
handler.InitConfig();
// create mock cgroup
FilePath cgroupMountDir = CreateMockCgroupMount(cgroupDir);
// create mock mtab
FilePath mockMtab = CreateMockMTab(cgroupDir);
// setup our handler and call init()
handler.SetMtabFile(mockMtab.GetAbsolutePath());
// check values
// in this case, we're using all cpu so the files
// shouldn't exist(because init won't create them
handler.Init(mockLCE, plugin);
FilePath periodFile = new FilePath(cgroupMountDir, "cpu.cfs_period_us");
FilePath quotaFile = new FilePath(cgroupMountDir, "cpu.cfs_quota_us");
NUnit.Framework.Assert.IsFalse(periodFile.Exists());
NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
// subset of cpu being used, files should be created
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 75);
handler.limits[0] = 100 * 1000;
handler.limits[1] = 1000 * 1000;
handler.Init(mockLCE, plugin);
int period = ReadIntFromFile(periodFile);
int quota = ReadIntFromFile(quotaFile);
NUnit.Framework.Assert.AreEqual(100 * 1000, period);
NUnit.Framework.Assert.AreEqual(1000 * 1000, quota);
// set cpu back to 100, quota should be -1
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 100);
handler.limits[0] = 100 * 1000;
handler.limits[1] = 1000 * 1000;
handler.Init(mockLCE, plugin);
quota = ReadIntFromFile(quotaFile);
NUnit.Framework.Assert.AreEqual(-1, quota);
FileUtils.DeleteQuietly(cgroupDir);
}
public virtual void TestRunCommandWithCpuAndMemoryResources()
{
// Windows only test
Assume.AssumeTrue(Shell.Windows);
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmWindowsContainerCpuLimitEnabled, "true");
conf.Set(YarnConfiguration.NmWindowsContainerMemoryLimitEnabled, "true");
string[] command = containerExecutor.GetRunCommand("echo", "group1", null, null,
conf, Resource.NewInstance(1024, 1));
float yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(ResourceCalculatorPlugin
.GetResourceCalculatorPlugin(null, conf), conf);
int cpuRate = Math.Min(10000, (int)((1 * 10000) / yarnProcessors));
// Assert the cpu and memory limits are set correctly in the command
string[] expected = new string[] { Shell.Winutils, "task", "create", "-m", "1024"
, "-c", cpuRate.ToString(), "group1", "cmd /c " + "echo" };
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected, command));
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.monitoringInterval = conf.GetLong(YarnConfiguration.NmContainerMonIntervalMs
, YarnConfiguration.DefaultNmContainerMonIntervalMs);
Type clazz = conf.GetClass <ResourceCalculatorPlugin>(YarnConfiguration.NmContainerMonResourceCalculator
, null);
this.resourceCalculatorPlugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
(clazz, conf);
Log.Info(" Using ResourceCalculatorPlugin : " + this.resourceCalculatorPlugin);
processTreeClass = conf.GetClass <ResourceCalculatorProcessTree>(YarnConfiguration
.NmContainerMonProcessTree, null);
this.conf = conf;
Log.Info(" Using ResourceCalculatorProcessTree : " + this.processTreeClass);
this.containerMetricsEnabled = conf.GetBoolean(YarnConfiguration.NmContainerMetricsEnable
, YarnConfiguration.DefaultNmContainerMetricsEnable);
this.containerMetricsPeriodMs = conf.GetLong(YarnConfiguration.NmContainerMetricsPeriodMs
, YarnConfiguration.DefaultNmContainerMetricsPeriodMs);
this.containerMetricsUnregisterDelayMs = conf.GetLong(YarnConfiguration.NmContainerMetricsUnregisterDelayMs
, YarnConfiguration.DefaultNmContainerMetricsUnregisterDelayMs);
long configuredPMemForContainers = conf.GetLong(YarnConfiguration.NmPmemMb, YarnConfiguration
.DefaultNmPmemMb) * 1024 * 1024l;
long configuredVCoresForContainers = conf.GetLong(YarnConfiguration.NmVcores, YarnConfiguration
.DefaultNmVcores);
// Setting these irrespective of whether checks are enabled. Required in
// the UI.
// ///////// Physical memory configuration //////
this.maxPmemAllottedForContainers = configuredPMemForContainers;
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
float vmemRatio = conf.GetFloat(YarnConfiguration.NmVmemPmemRatio, YarnConfiguration
.DefaultNmVmemPmemRatio);
Preconditions.CheckArgument(vmemRatio > 0.99f, YarnConfiguration.NmVmemPmemRatio
+ " should be at least 1.0");
this.maxVmemAllottedForContainers = (long)(vmemRatio * configuredPMemForContainers
);
pmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmPmemCheckEnabled, YarnConfiguration
.DefaultNmPmemCheckEnabled);
vmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmVmemCheckEnabled, YarnConfiguration
.DefaultNmVmemCheckEnabled);
Log.Info("Physical memory check enabled: " + pmemCheckEnabled);
Log.Info("Virtual memory check enabled: " + vmemCheckEnabled);
nodeCpuPercentageForYARN = NodeManagerHardwareUtils.GetNodeCpuPercentage(conf);
if (pmemCheckEnabled)
{
// Logging if actual pmem cannot be determined.
long totalPhysicalMemoryOnNM = UnknownMemoryLimit;
if (this.resourceCalculatorPlugin != null)
{
totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin.GetPhysicalMemorySize();
if (totalPhysicalMemoryOnNM <= 0)
{
Log.Warn("NodeManager's totalPmem could not be calculated. " + "Setting it to " +
UnknownMemoryLimit);
totalPhysicalMemoryOnNM = UnknownMemoryLimit;
}
}
if (totalPhysicalMemoryOnNM != UnknownMemoryLimit && this.maxPmemAllottedForContainers
> totalPhysicalMemoryOnNM * 0.80f)
{
Log.Warn("NodeManager configured with " + StringUtils.TraditionalBinaryPrefix.Long2String
(maxPmemAllottedForContainers, string.Empty, 1) + " physical memory allocated to containers, which is more than "
+ "80% of the total physical memory available (" + StringUtils.TraditionalBinaryPrefix
.Long2String(totalPhysicalMemoryOnNM, string.Empty, 1) + "). Thrashing might happen."
);
}
}
base.ServiceInit(conf);
}
public virtual void TestContainerLimits()
{
LinuxContainerExecutor mockLCE = new TestCgroupsLCEResourcesHandler.MockLinuxContainerExecutor
();
TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler handler = new TestCgroupsLCEResourcesHandler.CustomCgroupsLCEResourceHandler
();
handler.generateLimitsMode = true;
YarnConfiguration conf = new YarnConfiguration();
int numProcessors = 4;
ResourceCalculatorPlugin plugin = Org.Mockito.Mockito.Mock <ResourceCalculatorPlugin
>();
Org.Mockito.Mockito.DoReturn(numProcessors).When(plugin).GetNumProcessors();
handler.SetConf(conf);
handler.InitConfig();
// create mock cgroup
FilePath cgroupMountDir = CreateMockCgroupMount(cgroupDir);
// create mock mtab
FilePath mockMtab = CreateMockMTab(cgroupDir);
// setup our handler and call init()
handler.SetMtabFile(mockMtab.GetAbsolutePath());
handler.Init(mockLCE, plugin);
// check values
// default case - files shouldn't exist, strict mode off by default
ContainerId id = ContainerId.FromString("container_1_1_1_1");
handler.PreExecute(id, Resource.NewInstance(1024, 1));
FilePath containerDir = new FilePath(cgroupMountDir, id.ToString());
NUnit.Framework.Assert.IsTrue(containerDir.Exists());
NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
FilePath periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
FilePath quotaFile = new FilePath(containerDir, "cpu.cfs_quota_us");
NUnit.Framework.Assert.IsFalse(periodFile.Exists());
NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
// no files created because we're using all cpu
FileUtils.DeleteQuietly(containerDir);
conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
);
handler.InitConfig();
handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
));
NUnit.Framework.Assert.IsTrue(containerDir.Exists());
NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
quotaFile = new FilePath(containerDir, "cpu.cfs_quota_us");
NUnit.Framework.Assert.IsFalse(periodFile.Exists());
NUnit.Framework.Assert.IsFalse(quotaFile.Exists());
// 50% of CPU
FileUtils.DeleteQuietly(containerDir);
conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
);
handler.InitConfig();
handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
/ 2));
NUnit.Framework.Assert.IsTrue(containerDir.Exists());
NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
quotaFile = new FilePath(containerDir, "cpu.cfs_quota_us");
NUnit.Framework.Assert.IsTrue(periodFile.Exists());
NUnit.Framework.Assert.IsTrue(quotaFile.Exists());
NUnit.Framework.Assert.AreEqual(500 * 1000, ReadIntFromFile(periodFile));
NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(quotaFile));
// CGroups set to 50% of CPU, container set to 50% of YARN CPU
FileUtils.DeleteQuietly(containerDir);
conf.SetBoolean(YarnConfiguration.NmLinuxContainerCgroupsStrictResourceUsage, true
);
conf.SetInt(YarnConfiguration.NmResourcePercentagePhysicalCpuLimit, 50);
handler.InitConfig();
handler.Init(mockLCE, plugin);
handler.PreExecute(id, Resource.NewInstance(1024, YarnConfiguration.DefaultNmVcores
/ 2));
NUnit.Framework.Assert.IsTrue(containerDir.Exists());
NUnit.Framework.Assert.IsTrue(containerDir.IsDirectory());
periodFile = new FilePath(containerDir, "cpu.cfs_period_us");
quotaFile = new FilePath(containerDir, "cpu.cfs_quota_us");
NUnit.Framework.Assert.IsTrue(periodFile.Exists());
NUnit.Framework.Assert.IsTrue(quotaFile.Exists());
NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(periodFile));
NUnit.Framework.Assert.AreEqual(1000 * 1000, ReadIntFromFile(quotaFile));
FileUtils.DeleteQuietly(cgroupDir);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Init(LinuxContainerExecutor lce)
{
this.Init(lce, ResourceCalculatorPlugin.GetResourceCalculatorPlugin(null, conf));
}