public static MetricsSystem MockMetricsSystem() { MetricsSystem ms = Org.Mockito.Mockito.Mock <MetricsSystem>(); DefaultMetricsSystem.SetInstance(ms); return(ms); }
public static Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics Create(RetryCache cache ) { Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics (cache); return(DefaultMetricsSystem.Instance().Register(m.name, null, m)); }
/// <exception cref="System.Exception"/> public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf , bool mapSpeculative, bool reduceSpeculative) { string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser( ).GetShortUserName()); conf.Set(MRJobConfig.UserName, user); conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString()); conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true); // TODO: fix the bug where the speculator gets events with // not-fully-constructed objects. For now, disable speculative exec conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative); conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative); Init(conf); Start(); DefaultMetricsSystem.Shutdown(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values .GetEnumerator().Next(); if (assignedQueue != null) { job.SetQueueName(assignedQueue); } // Write job.xml string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID() )); Log.Info("Writing job conf to " + jobFile); new FilePath(jobFile).GetParentFile().Mkdirs(); conf.WriteXml(new FileOutputStream(jobFile)); return(job); }
public override T Register <T>(string name, string desc, T source) { lock (this) { MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(source); MetricsSource s = sb.Build(); MetricsInfo si = sb.Info(); string name2 = name == null?si.Name() : name; string finalDesc = desc == null?si.Description() : desc; string finalName = DefaultMetricsSystem.SourceName(name2, !monitoring); // be friendly to non-metrics tests allSources[finalName] = s; Log.Debug(finalName + ", " + finalDesc); if (monitoring) { RegisterSource(finalName, finalDesc, s); } // We want to re-register the source to pick up new config when the // metrics system restarts. Register(finalName, new _AbstractCallback_238(this, finalName, finalDesc, s)); return(source); } }
public static Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics Create(Server server, Configuration conf) { Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics (server, conf); return(DefaultMetricsSystem.Instance().Register(m.name, null, m)); }
public static Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics Create(Journal j) { Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics m = new Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics (j); return(DefaultMetricsSystem.Instance().Register(m.GetName(), null, m)); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true); rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled , YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled); InitAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager (conf, nmStore); NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore ); RecoverTokens(nmTokenSecretManager, containerTokenSecretManager); this.aclsManager = new ApplicationACLsManager(conf); ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor >(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf ); try { exec.Init(); } catch (IOException e) { throw new YarnRuntimeException("Failed to initialize container executor", e); } DeletionService del = CreateDeletionService(exec); AddService(del); // NodeManager level dispatcher this.dispatcher = new AsyncDispatcher(); nodeHealthChecker = new NodeHealthCheckerService(); AddService(nodeHealthChecker); dirsHandler = nodeHealthChecker.GetDiskHandler(); this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager, nmStore); nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker ); NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor(); AddService(nodeResourceMonitor); containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater, this.aclsManager, dirsHandler); AddService(containerManager); ((NodeManager.NMContext)context).SetContainerManager(containerManager); WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor (), this.aclsManager, dirsHandler); AddService(webServer); ((NodeManager.NMContext)context).SetWebServer(webServer); dispatcher.Register(typeof(ContainerManagerEventType), containerManager); dispatcher.Register(typeof(NodeManagerEventType), this); AddService(dispatcher); DefaultMetricsSystem.Initialize("NodeManager"); // StatusUpdater should be added last so that it get started last // so that we make sure everything is up before registering with RM. AddService(nodeStatusUpdater); base.ServiceInit(conf); }
/// <exception cref="System.Exception"/> protected override void ServiceStop() { if (webApp != null) { webApp.Stop(); } DefaultMetricsSystem.Shutdown(); base.ServiceStop(); }
ForQueue(string queueName, Queue parent, bool enableUserMetrics, Configuration conf) { lock (typeof(QueueMetrics)) { return(ForQueue(DefaultMetricsSystem.Instance(), queueName, parent, enableUserMetrics , conf)); } }
public virtual void TestInitFirstVerifyStopInvokedImmediately() { DefaultMetricsSystem.Shutdown(); new ConfigBuilder().Add("*.period", 8).Add("test.sink.test.class", typeof(TestMetricsSystemImpl.TestSink ).FullName).Add("test.*.source.filter.exclude", "s0").Add("test.source.s1.metric.filter.exclude" , "X*").Add("test.sink.sink1.metric.filter.exclude", "Y*").Add("test.sink.sink2.metric.filter.exclude" , "Y*").Save(TestMetricsConfig.GetTestFilename("hadoop-metrics2-test")); //.add("test.sink.plugin.urls", getPluginUrlsAsString()) MetricsSystemImpl ms = new MetricsSystemImpl("Test"); ms.Start(); ms.Register("s0", "s0 desc", new TestMetricsSystemImpl.TestSource("s0rec")); TestMetricsSystemImpl.TestSource s1 = ms.Register("s1", "s1 desc", new TestMetricsSystemImpl.TestSource ("s1rec")); s1.c1.Incr(); s1.xxx.Incr(); s1.g1.Set(2); s1.yyy.Incr(2); s1.s1.Add(0); MetricsSink sink1 = Org.Mockito.Mockito.Mock <MetricsSink>(); MetricsSink sink2 = Org.Mockito.Mockito.Mock <MetricsSink>(); ms.RegisterSink("sink1", "sink1 desc", sink1); ms.RegisterSink("sink2", "sink2 desc", sink2); ms.PublishMetricsNow(); // publish the metrics ms.Stop(); ms.Shutdown(); //When we call stop, at most two sources will be consumed by each sink thread. Org.Mockito.Mockito.Verify(sink1, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r1.Capture ()); IList <MetricsRecord> mr1 = r1.GetAllValues(); Org.Mockito.Mockito.Verify(sink2, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r2.Capture ()); IList <MetricsRecord> mr2 = r2.GetAllValues(); if (mr1.Count != 0 && mr2.Count != 0) { CheckMetricsRecords(mr1); MoreAsserts.AssertEquals("output", mr1, mr2); } else { if (mr1.Count != 0) { CheckMetricsRecords(mr1); } else { if (mr2.Count != 0) { CheckMetricsRecords(mr2); } } } }
public virtual void TearDown() { ClusterMetrics.Destroy(); MetricsSystem ms = DefaultMetricsSystem.Instance(); if (ms.GetSource("ClusterMetrics") != null) { DefaultMetricsSystem.Shutdown(); } }
public static void Setup() { //Do not error out if metrics are inited multiple times DefaultMetricsSystem.SetMiniClusterMode(true); FilePath dir = new FilePath(stagingDir); stagingDir = dir.GetAbsolutePath(); localFS = FileContext.GetLocalFSFileContext(); localFS.Delete(new Path(testDir.GetAbsolutePath()), true); testDir.Mkdir(); }
Create() { MetricsSystem ms = DefaultMetricsSystem.Instance(); Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics metrics = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics (); ms.Register("SharedCacheUploaderRequests", null, metrics); return(metrics); }
private static void RegisterMetrics() { registry = new MetricsRegistry(RecordInfo); registry.Tag(RecordInfo, "ResourceManager"); MetricsSystem ms = DefaultMetricsSystem.Instance(); if (ms != null) { ms.Register("ClusterMetrics", "Metrics for the Yarn Cluster", Instance); } }
private FSOpDurations() { registry = new MetricsRegistry(RecordInfo); registry.Tag(RecordInfo, "FSOpDurations"); MetricsSystem ms = DefaultMetricsSystem.Instance(); if (ms != null) { ms.Register(RecordInfo.Name(), RecordInfo.Description(), this); } }
public virtual void TearDown() { MetricsSource source = DefaultMetricsSystem.Instance().GetSource("UgiMetrics"); if (source != null) { // Run only once since the UGI metrics is cleaned up during teardown MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(source); MetricsAsserts.AssertQuantileGauges("GetGroups1s", rb); } cluster.Shutdown(); }
internal JvmMetrics Init(string processName, string sessionId) { lock (this) { if (JvmMetrics.Singleton.impl == null) { JvmMetrics.Singleton.impl = Create(processName, sessionId, DefaultMetricsSystem.Instance ()); } return(JvmMetrics.Singleton.impl); } }
public static Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics Create(Configuration conf , string gatewayName) { string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey); MetricsSystem ms = DefaultMetricsSystem.Instance(); JvmMetrics jm = JvmMetrics.Create(gatewayName, sessionId, ms); // Percentile measurement is [50th,75th,90th,95th,99th] currently int[] intervals = conf.GetInts(NfsConfigKeys.NfsMetricsPercentilesIntervalsKey); return(ms.Register(new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics(gatewayName, sessionId , intervals, jm))); }
public static Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics Create (Configuration conf, HdfsServerConstants.NamenodeRole r) { string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey); string processName = r.ToString(); MetricsSystem ms = DefaultMetricsSystem.Instance(); JvmMetrics jm = JvmMetrics.Create(processName, sessionId, ms); // Percentile measurement is off by default, by watching no intervals int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey); return(ms.Register(new Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics (processName, sessionId, intervals, jm))); }
Create() { MetricsSystem ms = DefaultMetricsSystem.Instance(); Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics metricObject = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics(); MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(metricObject); MetricsSource s = sb.Build(); ms.Register("cleaner", "The cleaner service of truly shared cache", s); metricObject.metricSource = s; return(metricObject); }
public static Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics Create (Configuration conf, string dnName) { string sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey); MetricsSystem ms = DefaultMetricsSystem.Instance(); JvmMetrics jm = JvmMetrics.Create("DataNode", sessionId, ms); string name = "DataNodeActivity-" + (dnName.IsEmpty() ? "UndefinedDataNodeName" + DFSUtil.GetRandom().Next() : dnName.Replace(':', '-')); // Percentile measurement is off by default, by watching no intervals int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey); return(ms.Register(name, null, new Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics (name, sessionId, intervals, jm))); }
private static ObjectName GetMBeanName(string serviceName, string nameName) { ObjectName name = null; string nameStr = "Hadoop:service=" + serviceName + ",name=" + nameName; try { name = DefaultMetricsSystem.NewMBeanName(nameStr); } catch (Exception e) { Log.Warn("Error creating MBean object name: " + nameStr, e); } return(name); }
/// <summary>Initialized the metrics system with a prefix.</summary> /// <param name="prefix">the system will look for configs with the prefix</param> /// <returns>the metrics system object itself</returns> public override MetricsSystem Init(string prefix) { lock (this) { if (monitoring && !DefaultMetricsSystem.InMiniClusterMode()) { Log.Warn(this.prefix + " metrics system already initialized!"); return(this); } this.prefix = Preconditions.CheckNotNull(prefix, "prefix"); ++refCount; if (monitoring) { // in mini cluster mode Log.Info(this.prefix + " metrics system started (again)"); return(this); } switch (InitMode()) { case MetricsSystemImpl.InitMode.Normal: { try { Start(); } catch (MetricsConfigException e) { // Configuration errors (e.g., typos) should not be fatal. // We can always start the metrics system later via JMX. Log.Warn("Metrics system not started: " + e.Message); Log.Debug("Stacktrace: ", e); } break; } case MetricsSystemImpl.InitMode.Standby: { Log.Info(prefix + " metrics system started in standby mode"); break; } } InitSystemMBean(); return(this); } }
/// <exception cref="System.Exception"/> protected override void ServiceStop() { if (isStopping.GetAndSet(true)) { return; } try { base.ServiceStop(); DefaultMetricsSystem.Shutdown(); } finally { // YARN-3641: NM's services stop get failed shouldn't block the // release of NMLevelDBStore. StopRecoveryStore(); } }
/// <summary>Start listening for edits via RPC.</summary> /// <exception cref="System.IO.IOException"/> public virtual void Start() { Preconditions.CheckState(!IsStarted(), "JN already running"); ValidateAndCreateJournalDir(localDir); DefaultMetricsSystem.Initialize("JournalNode"); JvmMetrics.Create("JournalNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey), DefaultMetricsSystem.Instance()); IPEndPoint socAddr = JournalNodeRpcServer.GetAddress(conf); SecurityUtil.Login(conf, DFSConfigKeys.DfsJournalnodeKeytabFileKey, DFSConfigKeys .DfsJournalnodeKerberosPrincipalKey, socAddr.GetHostName()); RegisterJNMXBean(); httpServer = new JournalNodeHttpServer(conf, this); httpServer.Start(); httpServerURI = httpServer.GetServerURI().ToString(); rpcServer = new JournalNodeRpcServer(conf, this); rpcServer.Start(); }
public virtual void TearDown() { if (hostFile != null && hostFile.Exists()) { hostFile.Delete(); } ClusterMetrics.Destroy(); if (rm != null) { rm.Stop(); } MetricsSystem ms = DefaultMetricsSystem.Instance(); if (ms.GetSource("ClusterMetrics") != null) { DefaultMetricsSystem.Shutdown(); } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { // init timeline services first timelineStore = CreateTimelineStore(conf); AddIfService(timelineStore); secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf); AddService(secretManagerService); timelineDataManager = CreateTimelineDataManager(conf); AddService(timelineDataManager); // init generic history service afterwards aclsManager = CreateApplicationACLsManager(conf); historyManager = CreateApplicationHistoryManager(conf); ahsClientService = CreateApplicationHistoryClientService(historyManager); AddService(ahsClientService); AddService((Org.Apache.Hadoop.Service.Service)historyManager); DefaultMetricsSystem.Initialize("ApplicationHistoryServer"); JvmMetrics.InitSingleton("ApplicationHistoryServer", null); base.ServiceInit(conf); }
public virtual void TestNames() { DefaultMetricsSystem.Initialize("NodeManager"); NodeManagerMetrics metrics = NodeManagerMetrics.Create(); Resource total = Records.NewRecord <Resource>(); total.SetMemory(8 * GiB); total.SetVirtualCores(16); Resource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>(); resource.SetMemory(512); //512MiB resource.SetVirtualCores(2); metrics.AddResource(total); for (int i = 10; i-- > 0;) { // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB) metrics.LaunchedContainer(); metrics.AllocateContainer(resource); } metrics.InitingContainer(); metrics.EndInitingContainer(); metrics.RunningContainer(); metrics.EndRunningContainer(); // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB) metrics.CompletedContainer(); metrics.ReleaseContainer(resource); metrics.FailedContainer(); metrics.ReleaseContainer(resource); metrics.KilledContainer(); metrics.ReleaseContainer(resource); metrics.InitingContainer(); metrics.RunningContainer(); NUnit.Framework.Assert.IsTrue(!metrics.containerLaunchDuration.Changed()); metrics.AddContainerLaunchDuration(1); NUnit.Framework.Assert.IsTrue(metrics.containerLaunchDuration.Changed()); // availableGB is expected to be floored, // while allocatedGB is expected to be ceiled. // allocatedGB: 3.5GB allocated memory is shown as 4GB // availableGB: 4.5GB available memory is shown as 4GB CheckMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2); }
public virtual void SetUp() { configuration = new Configuration(); UserGroupInformation.SetConfiguration(configuration); configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true); configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId); foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration )) { configuration.Set(HAUtil.AddSuffix(confKey, Rm1NodeId), Rm1Address); configuration.Set(HAUtil.AddSuffix(confKey, Rm2NodeId), Rm2Address); configuration.Set(HAUtil.AddSuffix(confKey, Rm3NodeId), Rm3Address); } // Enable webapp to test web-services also configuration.SetBoolean(MockRM.EnableWebapp, true); configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true); ClusterMetrics.Destroy(); QueueMetrics.ClearQueueMetrics(); DefaultMetricsSystem.Shutdown(); }
public static void Unregister(ObjectName mbeanName) { Log.Debug("Unregistering " + mbeanName); MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer(); if (mbeanName == null) { Log.Debug("Stacktrace: ", new Exception()); return; } try { mbs.UnregisterMBean(mbeanName); } catch (Exception e) { Log.Warn("Error unregistering " + mbeanName, e); } DefaultMetricsSystem.RemoveMBeanName(mbeanName); }