public virtual void TestContainerMetricsFlow()
{
string Err = "Error in number of records";
// Create a dummy MetricsSystem
MetricsSystem system = Org.Mockito.Mockito.Mock <MetricsSystem>();
Org.Mockito.Mockito.DoReturn(this).When(system).Register(Matchers.AnyString(), Matchers.AnyString
(), Matchers.Any());
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerMetrics metrics = ContainerMetrics.ForContainer(containerId, 100, 1);
metrics.RecordMemoryUsage(1024);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
metrics.Finished();
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
collector.Clear();
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 0, collector.GetRecords().Count);
}
public static MetricsSystem MockMetricsSystem()
{
MetricsSystem ms = Org.Mockito.Mockito.Mock <MetricsSystem>();
DefaultMetricsSystem.SetInstance(ms);
return(ms);
}
public virtual void TestContainerMetricsLimit()
{
string Err = "Error in number of records";
MetricsSystem system = Org.Mockito.Mockito.Mock <MetricsSystem>();
Org.Mockito.Mockito.DoReturn(this).When(system).Register(Matchers.AnyString(), Matchers.AnyString
(), Matchers.Any());
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerMetrics metrics = ContainerMetrics.ForContainer(containerId, 100, 1);
int anyPmemLimit = 1024;
int anyVmemLimit = 2048;
int anyVcores = 10;
string anyProcessId = "1234";
metrics.RecordResourceLimit(anyVmemLimit, anyPmemLimit, anyVcores);
metrics.RecordProcessId(anyProcessId);
Sharpen.Thread.Sleep(110);
metrics.GetMetrics(collector, true);
NUnit.Framework.Assert.AreEqual(Err, 1, collector.GetRecords().Count);
MetricsRecord record = collector.GetRecords()[0];
MetricsRecords.AssertTag(record, ContainerMetrics.ProcessidInfo.Name(), anyProcessId
);
MetricsRecords.AssertMetric(record, ContainerMetrics.PmemLimitMetricName, anyPmemLimit
);
MetricsRecords.AssertMetric(record, ContainerMetrics.VmemLimitMetricName, anyVmemLimit
);
MetricsRecords.AssertMetric(record, ContainerMetrics.VcoreLimitMetricName, anyVcores
);
collector.Clear();
}
public static MetricsSource UserSource(MetricsSystem ms, string queue, string user
)
{
MetricsSource s = ms.GetSource(QueueMetrics.SourceName(queue).Append(",user=").Append
(user).ToString());
return(s);
}
public virtual void TearDown()
{
ClusterMetrics.Destroy();
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
Create()
{
MetricsSystem ms = DefaultMetricsSystem.Instance();
Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
metrics = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
();
ms.Register("SharedCacheUploaderRequests", null, metrics);
return(metrics);
}
private FSOpDurations()
{
registry = new MetricsRegistry(RecordInfo);
registry.Tag(RecordInfo, "FSOpDurations");
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms != null)
{
ms.Register(RecordInfo.Name(), RecordInfo.Description(), this);
}
}
private static void RegisterMetrics()
{
registry = new MetricsRegistry(RecordInfo);
registry.Tag(RecordInfo, "ResourceManager");
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms != null)
{
ms.Register("ClusterMetrics", "Metrics for the Yarn Cluster", Instance);
}
}
public static Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics Create(Configuration conf
, string gatewayName)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create(gatewayName, sessionId, ms);
// Percentile measurement is [50th,75th,90th,95th,99th] currently
int[] intervals = conf.GetInts(NfsConfigKeys.NfsMetricsPercentilesIntervalsKey);
return(ms.Register(new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics(gatewayName, sessionId
, intervals, jm)));
}
public static Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics Create
(Configuration conf, HdfsServerConstants.NamenodeRole r)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
string processName = r.ToString();
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
return(ms.Register(new Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics
(processName, sessionId, intervals, jm)));
}
Create()
{
MetricsSystem ms = DefaultMetricsSystem.Instance();
Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics metricObject
= new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics();
MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(metricObject);
MetricsSource s = sb.Build();
ms.Register("cleaner", "The cleaner service of truly shared cache", s);
metricObject.metricSource = s;
return(metricObject);
}
protected internal QueueMetrics(MetricsSystem ms, string queueName, Queue parent,
bool enableUserMetrics, Configuration conf)
{
registry = new MetricsRegistry(RecordInfo);
this.queueName = queueName;
this.parent = parent != null?parent.GetMetrics() : null;
this.users = enableUserMetrics ? new Dictionary <string, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.QueueMetrics
>() : null;
metricsSystem = ms;
this.conf = conf;
runningTime = BuildBuckets(conf);
}
public static Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics Create
(Configuration conf, string dnName)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create("DataNode", sessionId, ms);
string name = "DataNodeActivity-" + (dnName.IsEmpty() ? "UndefinedDataNodeName" +
DFSUtil.GetRandom().Next() : dnName.Replace(':', '-'));
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
return(ms.Register(name, null, new Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics
(name, sessionId, intervals, jm)));
}
public virtual void TearDown()
{
if (hostFile != null && hostFile.Exists())
{
hostFile.Delete();
}
ClusterMetrics.Destroy();
if (rm != null)
{
rm.Stop();
}
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
ForContainer(MetricsSystem ms, ContainerId containerId, long flushPeriodMs, long
delayMs)
{
lock (typeof(ContainerMetrics))
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Monitor.ContainerMetrics
metrics = usageMetrics[containerId];
if (metrics == null)
{
metrics = new Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Monitor.ContainerMetrics
(ms, containerId, flushPeriodMs, delayMs).Tag(RecordInfo, containerId);
// Register with the MetricsSystems
if (ms != null)
{
metrics = ms.Register(SourceName(containerId), "Metrics for container: " + containerId
, metrics);
}
usageMetrics[containerId] = metrics;
}
return(metrics);
}
}
ForQueue(MetricsSystem ms, string queueName, Queue parent, bool enableUserMetrics
, Configuration conf)
{
lock (typeof(QueueMetrics))
{
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.QueueMetrics metrics = queueMetrics
[queueName];
if (metrics == null)
{
metrics = new Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.QueueMetrics
(ms, queueName, parent, enableUserMetrics, conf).Tag(QueueInfo, queueName);
// Register with the MetricsSystems
if (ms != null)
{
metrics = ms.Register(SourceName(queueName).ToString(), "Metrics for queue: " + queueName
, metrics);
}
queueMetrics[queueName] = metrics;
}
return(metrics);
}
}
internal ContainerMetrics(MetricsSystem ms, ContainerId containerId, long flushPeriodMs
, long delayMs)
{
// Use a multiplier of 1000 to avoid losing too much precision when
// converting to integers
// This tracks overall CPU percentage of the machine in terms of percentage
// of 1 core similar to top
// Thus if you use 2 cores completely out of 4 available cores this value
// will be 200
// Metrics publishing status
// true if period elapsed
// true if container finished
// unregister
// lazily initialized
// Create a timer to unregister container metrics,
// whose associated thread run as a daemon.
this.recordInfo = Interns.Info(SourceName(containerId), RecordInfo.Description());
this.registry = new MetricsRegistry(recordInfo);
this.metricsSystem = ms;
this.containerId = containerId;
this.flushPeriodMs = flushPeriodMs;
this.unregisterDelayMs = delayMs < 0 ? 0 : delayMs;
ScheduleTimerTaskIfRequired();
this.pMemMBsStat = registry.NewStat(PmemUsageMetricName, "Physical memory stats",
"Usage", "MBs", true);
this.cpuCoreUsagePercent = registry.NewStat(PhyCpuUsageMetricName, "Physical Cpu core percent usage stats"
, "Usage", "Percents", true);
this.milliVcoresUsed = registry.NewStat(VcoreUsageMetricName, "1000 times Vcore usage"
, "Usage", "MilliVcores", true);
this.pMemLimitMbs = registry.NewGauge(PmemLimitMetricName, "Physical memory limit in MBs"
, 0);
this.vMemLimitMbs = registry.NewGauge(VmemLimitMetricName, "Virtual memory limit in MBs"
, 0);
this.cpuVcoreLimit = registry.NewGauge(VcoreLimitMetricName, "CPU limit in number of vcores"
, 0);
}
public virtual TestFileSink.MyMetrics2 RegisterWith(MetricsSystem ms)
{
return(ms.Register("m2", null, this));
}
public static MRAppMetrics Create(MetricsSystem ms)
{
JvmMetrics.InitSingleton("MRAppMaster", null);
return(ms.Register(new MRAppMetrics()));
}
public static JvmMetrics Create(string processName, string sessionId, MetricsSystem
ms)
{
return(ms.Register(JvmMetricsInfo.JvmMetrics.ToString(), JvmMetricsInfo.JvmMetrics
.Description(), new JvmMetrics(processName, sessionId)));
}
public static MetricsSystem SetInstance(MetricsSystem ms)
{
return DefaultMetricsSystem.Instance.SetImpl(ms);
}
internal MetricsSystem SetImpl(MetricsSystem ms)
{
return DefaultMetricsSystem.impl.GetAndSet(ms);
}
public static MetricsSource QueueSource(MetricsSystem ms, string queue)
{
MetricsSource s = ms.GetSource(QueueMetrics.SourceName(queue).ToString());
return(s);
}
public virtual void SetUp()
{
ms = new MetricsSystemImpl();
QueueMetrics.ClearQueueMetrics();
}
internal static NodeManagerMetrics Create(MetricsSystem ms)
{
JvmMetrics.Create("NodeManager", null, ms);
return(ms.Register(new NodeManagerMetrics()));
}
internal ShuffleHandler(MetricsSystem ms)
: base("httpshuffle")
{
metrics = ms.Register(new ShuffleHandler.ShuffleMetrics());
}
internal FSQueueMetrics(MetricsSystem ms, string queueName, Queue parent, bool enableUserMetrics
, Configuration conf)
: base(ms, queueName, parent, enableUserMetrics, conf)
{
}
