/// <summary>Returns the fraction of CPU cores that should be used for YARN containers.
/// </summary>
/// <remarks>
/// Returns the fraction of CPU cores that should be used for YARN containers.
/// The number is derived based on various configuration params such as
/// YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
/// </remarks>
/// <param name="conf">- Configuration object</param>
/// <returns>Fraction of CPU cores to be used for YARN containers</returns>
public static float GetContainersCores(Configuration conf)
{
ResourceCalculatorPlugin plugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
(null, conf);
return(NodeManagerHardwareUtils.GetContainersCores(plugin, conf));
}
public virtual void TestRunCommandWithCpuAndMemoryResources()
{
// Windows only test
Assume.AssumeTrue(Shell.Windows);
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmWindowsContainerCpuLimitEnabled, "true");
conf.Set(YarnConfiguration.NmWindowsContainerMemoryLimitEnabled, "true");
string[] command = containerExecutor.GetRunCommand("echo", "group1", null, null,
conf, Resource.NewInstance(1024, 1));
float yarnProcessors = NodeManagerHardwareUtils.GetContainersCores(ResourceCalculatorPlugin
.GetResourceCalculatorPlugin(null, conf), conf);
int cpuRate = Math.Min(10000, (int)((1 * 10000) / yarnProcessors));
// Assert the cpu and memory limits are set correctly in the command
string[] expected = new string[] { Shell.Winutils, "task", "create", "-m", "1024"
, "-c", cpuRate.ToString(), "group1", "cmd /c " + "echo" };
NUnit.Framework.Assert.IsTrue(Arrays.Equals(expected, command));
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.monitoringInterval = conf.GetLong(YarnConfiguration.NmContainerMonIntervalMs
, YarnConfiguration.DefaultNmContainerMonIntervalMs);
Type clazz = conf.GetClass <ResourceCalculatorPlugin>(YarnConfiguration.NmContainerMonResourceCalculator
, null);
this.resourceCalculatorPlugin = ResourceCalculatorPlugin.GetResourceCalculatorPlugin
(clazz, conf);
Log.Info(" Using ResourceCalculatorPlugin : " + this.resourceCalculatorPlugin);
processTreeClass = conf.GetClass <ResourceCalculatorProcessTree>(YarnConfiguration
.NmContainerMonProcessTree, null);
this.conf = conf;
Log.Info(" Using ResourceCalculatorProcessTree : " + this.processTreeClass);
this.containerMetricsEnabled = conf.GetBoolean(YarnConfiguration.NmContainerMetricsEnable
, YarnConfiguration.DefaultNmContainerMetricsEnable);
this.containerMetricsPeriodMs = conf.GetLong(YarnConfiguration.NmContainerMetricsPeriodMs
, YarnConfiguration.DefaultNmContainerMetricsPeriodMs);
this.containerMetricsUnregisterDelayMs = conf.GetLong(YarnConfiguration.NmContainerMetricsUnregisterDelayMs
, YarnConfiguration.DefaultNmContainerMetricsUnregisterDelayMs);
long configuredPMemForContainers = conf.GetLong(YarnConfiguration.NmPmemMb, YarnConfiguration
.DefaultNmPmemMb) * 1024 * 1024l;
long configuredVCoresForContainers = conf.GetLong(YarnConfiguration.NmVcores, YarnConfiguration
.DefaultNmVcores);
// Setting these irrespective of whether checks are enabled. Required in
// the UI.
// ///////// Physical memory configuration //////
this.maxPmemAllottedForContainers = configuredPMemForContainers;
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
float vmemRatio = conf.GetFloat(YarnConfiguration.NmVmemPmemRatio, YarnConfiguration
.DefaultNmVmemPmemRatio);
Preconditions.CheckArgument(vmemRatio > 0.99f, YarnConfiguration.NmVmemPmemRatio
+ " should be at least 1.0");
this.maxVmemAllottedForContainers = (long)(vmemRatio * configuredPMemForContainers
);
pmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmPmemCheckEnabled, YarnConfiguration
.DefaultNmPmemCheckEnabled);
vmemCheckEnabled = conf.GetBoolean(YarnConfiguration.NmVmemCheckEnabled, YarnConfiguration
.DefaultNmVmemCheckEnabled);
Log.Info("Physical memory check enabled: " + pmemCheckEnabled);
Log.Info("Virtual memory check enabled: " + vmemCheckEnabled);
nodeCpuPercentageForYARN = NodeManagerHardwareUtils.GetNodeCpuPercentage(conf);
if (pmemCheckEnabled)
{
// Logging if actual pmem cannot be determined.
long totalPhysicalMemoryOnNM = UnknownMemoryLimit;
if (this.resourceCalculatorPlugin != null)
{
totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin.GetPhysicalMemorySize();
if (totalPhysicalMemoryOnNM <= 0)
{
Log.Warn("NodeManager's totalPmem could not be calculated. " + "Setting it to " +
UnknownMemoryLimit);
totalPhysicalMemoryOnNM = UnknownMemoryLimit;
}
}
if (totalPhysicalMemoryOnNM != UnknownMemoryLimit && this.maxPmemAllottedForContainers
> totalPhysicalMemoryOnNM * 0.80f)
{
Log.Warn("NodeManager configured with " + StringUtils.TraditionalBinaryPrefix.Long2String
(maxPmemAllottedForContainers, string.Empty, 1) + " physical memory allocated to containers, which is more than "
+ "80% of the total physical memory available (" + StringUtils.TraditionalBinaryPrefix
.Long2String(totalPhysicalMemoryOnNM, string.Empty, 1) + "). Thrashing might happen."
);
}
}
base.ServiceInit(conf);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Init(LinuxContainerExecutor lce)
{
this.Init(lce, ResourceCalculatorPlugin.GetResourceCalculatorPlugin(null, conf));
}
