public static MetricsSystem MockMetricsSystem()
{
MetricsSystem ms = Org.Mockito.Mockito.Mock <MetricsSystem>();
DefaultMetricsSystem.SetInstance(ms);
return(ms);
}
public static Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics Create(RetryCache cache
)
{
Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics
(cache);
return(DefaultMetricsSystem.Instance().Register(m.name, null, m));
}
/// <exception cref="System.Exception"/>
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf
, bool mapSpeculative, bool reduceSpeculative)
{
string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser(
).GetShortUserName());
conf.Set(MRJobConfig.UserName, user);
conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString());
conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true);
// TODO: fix the bug where the speculator gets events with
// not-fully-constructed objects. For now, disable speculative exec
conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative);
conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative);
Init(conf);
Start();
DefaultMetricsSystem.Shutdown();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values
.GetEnumerator().Next();
if (assignedQueue != null)
{
job.SetQueueName(assignedQueue);
}
// Write job.xml
string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID()
));
Log.Info("Writing job conf to " + jobFile);
new FilePath(jobFile).GetParentFile().Mkdirs();
conf.WriteXml(new FileOutputStream(jobFile));
return(job);
}
public override T Register <T>(string name, string desc, T source)
{
lock (this)
{
MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(source);
MetricsSource s = sb.Build();
MetricsInfo si = sb.Info();
string name2 = name == null?si.Name() : name;
string finalDesc = desc == null?si.Description() : desc;
string finalName = DefaultMetricsSystem.SourceName(name2, !monitoring);
// be friendly to non-metrics tests
allSources[finalName] = s;
Log.Debug(finalName + ", " + finalDesc);
if (monitoring)
{
RegisterSource(finalName, finalDesc, s);
}
// We want to re-register the source to pick up new config when the
// metrics system restarts.
Register(finalName, new _AbstractCallback_238(this, finalName, finalDesc, s));
return(source);
}
}
public static Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics Create(Server server, Configuration
conf)
{
Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics
(server, conf);
return(DefaultMetricsSystem.Instance().Register(m.name, null, m));
}
public static Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics Create(Journal
j)
{
Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics m = new Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics
(j);
return(DefaultMetricsSystem.Instance().Register(m.GetName(), null, m));
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled
, YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled);
InitAndStartRecoveryStore(conf);
NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager
(conf, nmStore);
NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore
);
RecoverTokens(nmTokenSecretManager, containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor
>(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf
);
try
{
exec.Init();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to initialize container executor", e);
}
DeletionService del = CreateDeletionService(exec);
AddService(del);
// NodeManager level dispatcher
this.dispatcher = new AsyncDispatcher();
nodeHealthChecker = new NodeHealthCheckerService();
AddService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.GetDiskHandler();
this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager,
nmStore);
nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker
);
NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor();
AddService(nodeResourceMonitor);
containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater,
this.aclsManager, dirsHandler);
AddService(containerManager);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor
(), this.aclsManager, dirsHandler);
AddService(webServer);
((NodeManager.NMContext)context).SetWebServer(webServer);
dispatcher.Register(typeof(ContainerManagerEventType), containerManager);
dispatcher.Register(typeof(NodeManagerEventType), this);
AddService(dispatcher);
DefaultMetricsSystem.Initialize("NodeManager");
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
AddService(nodeStatusUpdater);
base.ServiceInit(conf);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (webApp != null)
{
webApp.Stop();
}
DefaultMetricsSystem.Shutdown();
base.ServiceStop();
}
ForQueue(string queueName, Queue parent, bool enableUserMetrics, Configuration
conf)
{
lock (typeof(QueueMetrics))
{
return(ForQueue(DefaultMetricsSystem.Instance(), queueName, parent, enableUserMetrics
, conf));
}
}
public virtual void TestInitFirstVerifyStopInvokedImmediately()
{
DefaultMetricsSystem.Shutdown();
new ConfigBuilder().Add("*.period", 8).Add("test.sink.test.class", typeof(TestMetricsSystemImpl.TestSink
).FullName).Add("test.*.source.filter.exclude", "s0").Add("test.source.s1.metric.filter.exclude"
, "X*").Add("test.sink.sink1.metric.filter.exclude", "Y*").Add("test.sink.sink2.metric.filter.exclude"
, "Y*").Save(TestMetricsConfig.GetTestFilename("hadoop-metrics2-test"));
//.add("test.sink.plugin.urls", getPluginUrlsAsString())
MetricsSystemImpl ms = new MetricsSystemImpl("Test");
ms.Start();
ms.Register("s0", "s0 desc", new TestMetricsSystemImpl.TestSource("s0rec"));
TestMetricsSystemImpl.TestSource s1 = ms.Register("s1", "s1 desc", new TestMetricsSystemImpl.TestSource
("s1rec"));
s1.c1.Incr();
s1.xxx.Incr();
s1.g1.Set(2);
s1.yyy.Incr(2);
s1.s1.Add(0);
MetricsSink sink1 = Org.Mockito.Mockito.Mock <MetricsSink>();
MetricsSink sink2 = Org.Mockito.Mockito.Mock <MetricsSink>();
ms.RegisterSink("sink1", "sink1 desc", sink1);
ms.RegisterSink("sink2", "sink2 desc", sink2);
ms.PublishMetricsNow();
// publish the metrics
ms.Stop();
ms.Shutdown();
//When we call stop, at most two sources will be consumed by each sink thread.
Org.Mockito.Mockito.Verify(sink1, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r1.Capture
());
IList <MetricsRecord> mr1 = r1.GetAllValues();
Org.Mockito.Mockito.Verify(sink2, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r2.Capture
());
IList <MetricsRecord> mr2 = r2.GetAllValues();
if (mr1.Count != 0 && mr2.Count != 0)
{
CheckMetricsRecords(mr1);
MoreAsserts.AssertEquals("output", mr1, mr2);
}
else
{
if (mr1.Count != 0)
{
CheckMetricsRecords(mr1);
}
else
{
if (mr2.Count != 0)
{
CheckMetricsRecords(mr2);
}
}
}
}
public virtual void TearDown()
{
ClusterMetrics.Destroy();
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
public static void Setup()
{
//Do not error out if metrics are inited multiple times
DefaultMetricsSystem.SetMiniClusterMode(true);
FilePath dir = new FilePath(stagingDir);
stagingDir = dir.GetAbsolutePath();
localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(testDir.GetAbsolutePath()), true);
testDir.Mkdir();
}
Create()
{
MetricsSystem ms = DefaultMetricsSystem.Instance();
Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
metrics = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
();
ms.Register("SharedCacheUploaderRequests", null, metrics);
return(metrics);
}
private static void RegisterMetrics()
{
registry = new MetricsRegistry(RecordInfo);
registry.Tag(RecordInfo, "ResourceManager");
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms != null)
{
ms.Register("ClusterMetrics", "Metrics for the Yarn Cluster", Instance);
}
}
private FSOpDurations()
{
registry = new MetricsRegistry(RecordInfo);
registry.Tag(RecordInfo, "FSOpDurations");
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms != null)
{
ms.Register(RecordInfo.Name(), RecordInfo.Description(), this);
}
}
public virtual void TearDown()
{
MetricsSource source = DefaultMetricsSystem.Instance().GetSource("UgiMetrics");
if (source != null)
{
// Run only once since the UGI metrics is cleaned up during teardown
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(source);
MetricsAsserts.AssertQuantileGauges("GetGroups1s", rb);
}
cluster.Shutdown();
}
internal JvmMetrics Init(string processName, string sessionId)
{
lock (this)
{
if (JvmMetrics.Singleton.impl == null)
{
JvmMetrics.Singleton.impl = Create(processName, sessionId, DefaultMetricsSystem.Instance
());
}
return(JvmMetrics.Singleton.impl);
}
}
public static Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics Create(Configuration conf
, string gatewayName)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create(gatewayName, sessionId, ms);
// Percentile measurement is [50th,75th,90th,95th,99th] currently
int[] intervals = conf.GetInts(NfsConfigKeys.NfsMetricsPercentilesIntervalsKey);
return(ms.Register(new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics(gatewayName, sessionId
, intervals, jm)));
}
public static Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics Create
(Configuration conf, HdfsServerConstants.NamenodeRole r)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
string processName = r.ToString();
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
return(ms.Register(new Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics
(processName, sessionId, intervals, jm)));
}
Create()
{
MetricsSystem ms = DefaultMetricsSystem.Instance();
Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics metricObject
= new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics();
MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(metricObject);
MetricsSource s = sb.Build();
ms.Register("cleaner", "The cleaner service of truly shared cache", s);
metricObject.metricSource = s;
return(metricObject);
}
public static Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics Create
(Configuration conf, string dnName)
{
string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
MetricsSystem ms = DefaultMetricsSystem.Instance();
JvmMetrics jm = JvmMetrics.Create("DataNode", sessionId, ms);
string name = "DataNodeActivity-" + (dnName.IsEmpty() ? "UndefinedDataNodeName" +
DFSUtil.GetRandom().Next() : dnName.Replace(':', '-'));
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
return(ms.Register(name, null, new Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics
(name, sessionId, intervals, jm)));
}
private static ObjectName GetMBeanName(string serviceName, string nameName)
{
ObjectName name = null;
string nameStr = "Hadoop:service=" + serviceName + ",name=" + nameName;
try
{
name = DefaultMetricsSystem.NewMBeanName(nameStr);
}
catch (Exception e)
{
Log.Warn("Error creating MBean object name: " + nameStr, e);
}
return(name);
}
/// <summary>Initialized the metrics system with a prefix.</summary>
/// <param name="prefix">the system will look for configs with the prefix</param>
/// <returns>the metrics system object itself</returns>
public override MetricsSystem Init(string prefix)
{
lock (this)
{
if (monitoring && !DefaultMetricsSystem.InMiniClusterMode())
{
Log.Warn(this.prefix + " metrics system already initialized!");
return(this);
}
this.prefix = Preconditions.CheckNotNull(prefix, "prefix");
++refCount;
if (monitoring)
{
// in mini cluster mode
Log.Info(this.prefix + " metrics system started (again)");
return(this);
}
switch (InitMode())
{
case MetricsSystemImpl.InitMode.Normal:
{
try
{
Start();
}
catch (MetricsConfigException e)
{
// Configuration errors (e.g., typos) should not be fatal.
// We can always start the metrics system later via JMX.
Log.Warn("Metrics system not started: " + e.Message);
Log.Debug("Stacktrace: ", e);
}
break;
}
case MetricsSystemImpl.InitMode.Standby:
{
Log.Info(prefix + " metrics system started in standby mode");
break;
}
}
InitSystemMBean();
return(this);
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (isStopping.GetAndSet(true))
{
return;
}
try
{
base.ServiceStop();
DefaultMetricsSystem.Shutdown();
}
finally
{
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
StopRecoveryStore();
}
}
/// <summary>Start listening for edits via RPC.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void Start()
{
Preconditions.CheckState(!IsStarted(), "JN already running");
ValidateAndCreateJournalDir(localDir);
DefaultMetricsSystem.Initialize("JournalNode");
JvmMetrics.Create("JournalNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey),
DefaultMetricsSystem.Instance());
IPEndPoint socAddr = JournalNodeRpcServer.GetAddress(conf);
SecurityUtil.Login(conf, DFSConfigKeys.DfsJournalnodeKeytabFileKey, DFSConfigKeys
.DfsJournalnodeKerberosPrincipalKey, socAddr.GetHostName());
RegisterJNMXBean();
httpServer = new JournalNodeHttpServer(conf, this);
httpServer.Start();
httpServerURI = httpServer.GetServerURI().ToString();
rpcServer = new JournalNodeRpcServer(conf, this);
rpcServer.Start();
}
public virtual void TearDown()
{
if (hostFile != null && hostFile.Exists())
{
hostFile.Delete();
}
ClusterMetrics.Destroy();
if (rm != null)
{
rm.Stop();
}
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
// init timeline services first
timelineStore = CreateTimelineStore(conf);
AddIfService(timelineStore);
secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf);
AddService(secretManagerService);
timelineDataManager = CreateTimelineDataManager(conf);
AddService(timelineDataManager);
// init generic history service afterwards
aclsManager = CreateApplicationACLsManager(conf);
historyManager = CreateApplicationHistoryManager(conf);
ahsClientService = CreateApplicationHistoryClientService(historyManager);
AddService(ahsClientService);
AddService((Org.Apache.Hadoop.Service.Service)historyManager);
DefaultMetricsSystem.Initialize("ApplicationHistoryServer");
JvmMetrics.InitSingleton("ApplicationHistoryServer", null);
base.ServiceInit(conf);
}
public virtual void TestNames()
{
DefaultMetricsSystem.Initialize("NodeManager");
NodeManagerMetrics metrics = NodeManagerMetrics.Create();
Resource total = Records.NewRecord <Resource>();
total.SetMemory(8 * GiB);
total.SetVirtualCores(16);
Resource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
resource.SetMemory(512);
//512MiB
resource.SetVirtualCores(2);
metrics.AddResource(total);
for (int i = 10; i-- > 0;)
{
// allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
metrics.LaunchedContainer();
metrics.AllocateContainer(resource);
}
metrics.InitingContainer();
metrics.EndInitingContainer();
metrics.RunningContainer();
metrics.EndRunningContainer();
// Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
metrics.CompletedContainer();
metrics.ReleaseContainer(resource);
metrics.FailedContainer();
metrics.ReleaseContainer(resource);
metrics.KilledContainer();
metrics.ReleaseContainer(resource);
metrics.InitingContainer();
metrics.RunningContainer();
NUnit.Framework.Assert.IsTrue(!metrics.containerLaunchDuration.Changed());
metrics.AddContainerLaunchDuration(1);
NUnit.Framework.Assert.IsTrue(metrics.containerLaunchDuration.Changed());
// availableGB is expected to be floored,
// while allocatedGB is expected to be ceiled.
// allocatedGB: 3.5GB allocated memory is shown as 4GB
// availableGB: 4.5GB available memory is shown as 4GB
CheckMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2);
}
public virtual void SetUp()
{
configuration = new Configuration();
UserGroupInformation.SetConfiguration(configuration);
configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true);
configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration
))
{
configuration.Set(HAUtil.AddSuffix(confKey, Rm1NodeId), Rm1Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm2NodeId), Rm2Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm3NodeId), Rm3Address);
}
// Enable webapp to test web-services also
configuration.SetBoolean(MockRM.EnableWebapp, true);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
ClusterMetrics.Destroy();
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}
public static void Unregister(ObjectName mbeanName)
{
Log.Debug("Unregistering " + mbeanName);
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
if (mbeanName == null)
{
Log.Debug("Stacktrace: ", new Exception());
return;
}
try
{
mbs.UnregisterMBean(mbeanName);
}
catch (Exception e)
{
Log.Warn("Error unregistering " + mbeanName, e);
}
DefaultMetricsSystem.RemoveMBeanName(mbeanName);
}