Ejemplo n.º 1
0
        public static MetricsSystem MockMetricsSystem()
        {
            MetricsSystem ms = Org.Mockito.Mockito.Mock <MetricsSystem>();

            DefaultMetricsSystem.SetInstance(ms);
            return(ms);
        }
Ejemplo n.º 2
0
 public static Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics Create(RetryCache cache
                                                                      )
 {
     Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RetryCacheMetrics
                                                             (cache);
     return(DefaultMetricsSystem.Instance().Register(m.name, null, m));
 }
Ejemplo n.º 3
0
        /// <exception cref="System.Exception"/>
        public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf
                                                                         , bool mapSpeculative, bool reduceSpeculative)
        {
            string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser(
                                       ).GetShortUserName());

            conf.Set(MRJobConfig.UserName, user);
            conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString());
            conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true);
            // TODO: fix the bug where the speculator gets events with
            // not-fully-constructed objects. For now, disable speculative exec
            conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative);
            conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative);
            Init(conf);
            Start();
            DefaultMetricsSystem.Shutdown();
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values
                                                             .GetEnumerator().Next();
            if (assignedQueue != null)
            {
                job.SetQueueName(assignedQueue);
            }
            // Write job.xml
            string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID()
                                                                                  ));

            Log.Info("Writing job conf to " + jobFile);
            new FilePath(jobFile).GetParentFile().Mkdirs();
            conf.WriteXml(new FileOutputStream(jobFile));
            return(job);
        }
Ejemplo n.º 4
0
        public override T Register <T>(string name, string desc, T source)
        {
            lock (this)
            {
                MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(source);
                MetricsSource        s  = sb.Build();
                MetricsInfo          si = sb.Info();
                string name2            = name == null?si.Name() : name;

                string finalDesc = desc == null?si.Description() : desc;

                string finalName = DefaultMetricsSystem.SourceName(name2, !monitoring);
                // be friendly to non-metrics tests
                allSources[finalName] = s;
                Log.Debug(finalName + ", " + finalDesc);
                if (monitoring)
                {
                    RegisterSource(finalName, finalDesc, s);
                }
                // We want to re-register the source to pick up new config when the
                // metrics system restarts.
                Register(finalName, new _AbstractCallback_238(this, finalName, finalDesc, s));
                return(source);
            }
        }
Ejemplo n.º 5
0
 public static Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics Create(Server server, Configuration
                                                               conf)
 {
     Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics m = new Org.Apache.Hadoop.Ipc.Metrics.RpcMetrics
                                                      (server, conf);
     return(DefaultMetricsSystem.Instance().Register(m.name, null, m));
 }
Ejemplo n.º 6
0
 public static Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics Create(Journal
                                                                            j)
 {
     Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics m = new Org.Apache.Hadoop.Hdfs.Qjournal.Server.JournalMetrics
                                                                   (j);
     return(DefaultMetricsSystem.Instance().Register(m.GetName(), null, m));
 }
Ejemplo n.º 7
0
        /// <exception cref="System.Exception"/>
        protected override void ServiceInit(Configuration conf)
        {
            conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
            rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled
                                                             , YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled);
            InitAndStartRecoveryStore(conf);
            NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager
                                                                            (conf, nmStore);
            NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore
                                                                                         );

            RecoverTokens(nmTokenSecretManager, containerTokenSecretManager);
            this.aclsManager = new ApplicationACLsManager(conf);
            ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor
                                                                                >(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf
                                                                 );

            try
            {
                exec.Init();
            }
            catch (IOException e)
            {
                throw new YarnRuntimeException("Failed to initialize container executor", e);
            }
            DeletionService del = CreateDeletionService(exec);

            AddService(del);
            // NodeManager level dispatcher
            this.dispatcher   = new AsyncDispatcher();
            nodeHealthChecker = new NodeHealthCheckerService();
            AddService(nodeHealthChecker);
            dirsHandler  = nodeHealthChecker.GetDiskHandler();
            this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager,
                                           nmStore);
            nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker
                                                        );
            NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor();

            AddService(nodeResourceMonitor);
            containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater,
                                                      this.aclsManager, dirsHandler);
            AddService(containerManager);
            ((NodeManager.NMContext)context).SetContainerManager(containerManager);
            WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor
                                                      (), this.aclsManager, dirsHandler);

            AddService(webServer);
            ((NodeManager.NMContext)context).SetWebServer(webServer);
            dispatcher.Register(typeof(ContainerManagerEventType), containerManager);
            dispatcher.Register(typeof(NodeManagerEventType), this);
            AddService(dispatcher);
            DefaultMetricsSystem.Initialize("NodeManager");
            // StatusUpdater should be added last so that it get started last
            // so that we make sure everything is up before registering with RM.
            AddService(nodeStatusUpdater);
            base.ServiceInit(conf);
        }
 /// <exception cref="System.Exception"/>
 protected override void ServiceStop()
 {
     if (webApp != null)
     {
         webApp.Stop();
     }
     DefaultMetricsSystem.Shutdown();
     base.ServiceStop();
 }
Ejemplo n.º 9
0
 ForQueue(string queueName, Queue parent, bool enableUserMetrics, Configuration
          conf)
 {
     lock (typeof(QueueMetrics))
     {
         return(ForQueue(DefaultMetricsSystem.Instance(), queueName, parent, enableUserMetrics
                         , conf));
     }
 }
Ejemplo n.º 10
0
        public virtual void TestInitFirstVerifyStopInvokedImmediately()
        {
            DefaultMetricsSystem.Shutdown();
            new ConfigBuilder().Add("*.period", 8).Add("test.sink.test.class", typeof(TestMetricsSystemImpl.TestSink
                                                                                      ).FullName).Add("test.*.source.filter.exclude", "s0").Add("test.source.s1.metric.filter.exclude"
                                                                                                                                                , "X*").Add("test.sink.sink1.metric.filter.exclude", "Y*").Add("test.sink.sink2.metric.filter.exclude"
                                                                                                                                                                                                               , "Y*").Save(TestMetricsConfig.GetTestFilename("hadoop-metrics2-test"));
            //.add("test.sink.plugin.urls", getPluginUrlsAsString())
            MetricsSystemImpl ms = new MetricsSystemImpl("Test");

            ms.Start();
            ms.Register("s0", "s0 desc", new TestMetricsSystemImpl.TestSource("s0rec"));
            TestMetricsSystemImpl.TestSource s1 = ms.Register("s1", "s1 desc", new TestMetricsSystemImpl.TestSource
                                                                  ("s1rec"));
            s1.c1.Incr();
            s1.xxx.Incr();
            s1.g1.Set(2);
            s1.yyy.Incr(2);
            s1.s1.Add(0);
            MetricsSink sink1 = Org.Mockito.Mockito.Mock <MetricsSink>();
            MetricsSink sink2 = Org.Mockito.Mockito.Mock <MetricsSink>();

            ms.RegisterSink("sink1", "sink1 desc", sink1);
            ms.RegisterSink("sink2", "sink2 desc", sink2);
            ms.PublishMetricsNow();
            // publish the metrics
            ms.Stop();
            ms.Shutdown();
            //When we call stop, at most two sources will be consumed by each sink thread.
            Org.Mockito.Mockito.Verify(sink1, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r1.Capture
                                                                                            ());
            IList <MetricsRecord> mr1 = r1.GetAllValues();

            Org.Mockito.Mockito.Verify(sink2, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r2.Capture
                                                                                            ());
            IList <MetricsRecord> mr2 = r2.GetAllValues();

            if (mr1.Count != 0 && mr2.Count != 0)
            {
                CheckMetricsRecords(mr1);
                MoreAsserts.AssertEquals("output", mr1, mr2);
            }
            else
            {
                if (mr1.Count != 0)
                {
                    CheckMetricsRecords(mr1);
                }
                else
                {
                    if (mr2.Count != 0)
                    {
                        CheckMetricsRecords(mr2);
                    }
                }
            }
        }
Ejemplo n.º 11
0
        public virtual void TearDown()
        {
            ClusterMetrics.Destroy();
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            if (ms.GetSource("ClusterMetrics") != null)
            {
                DefaultMetricsSystem.Shutdown();
            }
        }
Ejemplo n.º 12
0
		public static void Setup()
		{
			//Do not error out if metrics are inited multiple times
			DefaultMetricsSystem.SetMiniClusterMode(true);
			FilePath dir = new FilePath(stagingDir);
			stagingDir = dir.GetAbsolutePath();
			localFS = FileContext.GetLocalFSFileContext();
			localFS.Delete(new Path(testDir.GetAbsolutePath()), true);
			testDir.Mkdir();
		}
        Create()
        {
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
                metrics = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.SharedCacheUploaderMetrics
                              ();
            ms.Register("SharedCacheUploaderRequests", null, metrics);
            return(metrics);
        }
Ejemplo n.º 14
0
        private static void RegisterMetrics()
        {
            registry = new MetricsRegistry(RecordInfo);
            registry.Tag(RecordInfo, "ResourceManager");
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            if (ms != null)
            {
                ms.Register("ClusterMetrics", "Metrics for the Yarn Cluster", Instance);
            }
        }
Ejemplo n.º 15
0
        private FSOpDurations()
        {
            registry = new MetricsRegistry(RecordInfo);
            registry.Tag(RecordInfo, "FSOpDurations");
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            if (ms != null)
            {
                ms.Register(RecordInfo.Name(), RecordInfo.Description(), this);
            }
        }
Ejemplo n.º 16
0
        public virtual void TearDown()
        {
            MetricsSource source = DefaultMetricsSystem.Instance().GetSource("UgiMetrics");

            if (source != null)
            {
                // Run only once since the UGI metrics is cleaned up during teardown
                MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(source);
                MetricsAsserts.AssertQuantileGauges("GetGroups1s", rb);
            }
            cluster.Shutdown();
        }
Ejemplo n.º 17
0
 internal JvmMetrics Init(string processName, string sessionId)
 {
     lock (this)
     {
         if (JvmMetrics.Singleton.impl == null)
         {
             JvmMetrics.Singleton.impl = Create(processName, sessionId, DefaultMetricsSystem.Instance
                                                    ());
         }
         return(JvmMetrics.Singleton.impl);
     }
 }
Ejemplo n.º 18
0
        public static Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics Create(Configuration conf
                                                                         , string gatewayName)
        {
            string        sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
            MetricsSystem ms        = DefaultMetricsSystem.Instance();
            JvmMetrics    jm        = JvmMetrics.Create(gatewayName, sessionId, ms);

            // Percentile measurement is [50th,75th,90th,95th,99th] currently
            int[] intervals = conf.GetInts(NfsConfigKeys.NfsMetricsPercentilesIntervalsKey);
            return(ms.Register(new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3Metrics(gatewayName, sessionId
                                                                               , intervals, jm)));
        }
Ejemplo n.º 19
0
        public static Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics Create
            (Configuration conf, HdfsServerConstants.NamenodeRole r)
        {
            string        sessionId   = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
            string        processName = r.ToString();
            MetricsSystem ms          = DefaultMetricsSystem.Instance();
            JvmMetrics    jm          = JvmMetrics.Create(processName, sessionId, ms);

            // Percentile measurement is off by default, by watching no intervals
            int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
            return(ms.Register(new Org.Apache.Hadoop.Hdfs.Server.Namenode.Metrics.NameNodeMetrics
                                   (processName, sessionId, intervals, jm)));
        }
Ejemplo n.º 20
0
        Create()
        {
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics metricObject
                = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.Metrics.CleanerMetrics();
            MetricsSourceBuilder sb = MetricsAnnotations.NewSourceBuilder(metricObject);
            MetricsSource        s  = sb.Build();

            ms.Register("cleaner", "The cleaner service of truly shared cache", s);
            metricObject.metricSource = s;
            return(metricObject);
        }
Ejemplo n.º 21
0
        public static Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics Create
            (Configuration conf, string dnName)
        {
            string        sessionId = conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey);
            MetricsSystem ms        = DefaultMetricsSystem.Instance();
            JvmMetrics    jm        = JvmMetrics.Create("DataNode", sessionId, ms);
            string        name      = "DataNodeActivity-" + (dnName.IsEmpty() ? "UndefinedDataNodeName" +
                                                             DFSUtil.GetRandom().Next() : dnName.Replace(':', '-'));

            // Percentile measurement is off by default, by watching no intervals
            int[] intervals = conf.GetInts(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey);
            return(ms.Register(name, null, new Org.Apache.Hadoop.Hdfs.Server.Datanode.Metrics.DataNodeMetrics
                                   (name, sessionId, intervals, jm)));
        }
Ejemplo n.º 22
0
        private static ObjectName GetMBeanName(string serviceName, string nameName)
        {
            ObjectName name    = null;
            string     nameStr = "Hadoop:service=" + serviceName + ",name=" + nameName;

            try
            {
                name = DefaultMetricsSystem.NewMBeanName(nameStr);
            }
            catch (Exception e)
            {
                Log.Warn("Error creating MBean object name: " + nameStr, e);
            }
            return(name);
        }
Ejemplo n.º 23
0
        /// <summary>Initialized the metrics system with a prefix.</summary>
        /// <param name="prefix">the system will look for configs with the prefix</param>
        /// <returns>the metrics system object itself</returns>
        public override MetricsSystem Init(string prefix)
        {
            lock (this)
            {
                if (monitoring && !DefaultMetricsSystem.InMiniClusterMode())
                {
                    Log.Warn(this.prefix + " metrics system already initialized!");
                    return(this);
                }
                this.prefix = Preconditions.CheckNotNull(prefix, "prefix");
                ++refCount;
                if (monitoring)
                {
                    // in mini cluster mode
                    Log.Info(this.prefix + " metrics system started (again)");
                    return(this);
                }
                switch (InitMode())
                {
                case MetricsSystemImpl.InitMode.Normal:
                {
                    try
                    {
                        Start();
                    }
                    catch (MetricsConfigException e)
                    {
                        // Configuration errors (e.g., typos) should not be fatal.
                        // We can always start the metrics system later via JMX.
                        Log.Warn("Metrics system not started: " + e.Message);
                        Log.Debug("Stacktrace: ", e);
                    }
                    break;
                }

                case MetricsSystemImpl.InitMode.Standby:
                {
                    Log.Info(prefix + " metrics system started in standby mode");
                    break;
                }
                }
                InitSystemMBean();
                return(this);
            }
        }
Ejemplo n.º 24
0
 /// <exception cref="System.Exception"/>
 protected override void ServiceStop()
 {
     if (isStopping.GetAndSet(true))
     {
         return;
     }
     try
     {
         base.ServiceStop();
         DefaultMetricsSystem.Shutdown();
     }
     finally
     {
         // YARN-3641: NM's services stop get failed shouldn't block the
         // release of NMLevelDBStore.
         StopRecoveryStore();
     }
 }
Ejemplo n.º 25
0
        /// <summary>Start listening for edits via RPC.</summary>
        /// <exception cref="System.IO.IOException"/>
        public virtual void Start()
        {
            Preconditions.CheckState(!IsStarted(), "JN already running");
            ValidateAndCreateJournalDir(localDir);
            DefaultMetricsSystem.Initialize("JournalNode");
            JvmMetrics.Create("JournalNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey),
                              DefaultMetricsSystem.Instance());
            IPEndPoint socAddr = JournalNodeRpcServer.GetAddress(conf);

            SecurityUtil.Login(conf, DFSConfigKeys.DfsJournalnodeKeytabFileKey, DFSConfigKeys
                               .DfsJournalnodeKerberosPrincipalKey, socAddr.GetHostName());
            RegisterJNMXBean();
            httpServer = new JournalNodeHttpServer(conf, this);
            httpServer.Start();
            httpServerURI = httpServer.GetServerURI().ToString();
            rpcServer     = new JournalNodeRpcServer(conf, this);
            rpcServer.Start();
        }
Ejemplo n.º 26
0
        public virtual void TearDown()
        {
            if (hostFile != null && hostFile.Exists())
            {
                hostFile.Delete();
            }
            ClusterMetrics.Destroy();
            if (rm != null)
            {
                rm.Stop();
            }
            MetricsSystem ms = DefaultMetricsSystem.Instance();

            if (ms.GetSource("ClusterMetrics") != null)
            {
                DefaultMetricsSystem.Shutdown();
            }
        }
Ejemplo n.º 27
0
 /// <exception cref="System.Exception"/>
 protected override void ServiceInit(Configuration conf)
 {
     // init timeline services first
     timelineStore = CreateTimelineStore(conf);
     AddIfService(timelineStore);
     secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf);
     AddService(secretManagerService);
     timelineDataManager = CreateTimelineDataManager(conf);
     AddService(timelineDataManager);
     // init generic history service afterwards
     aclsManager      = CreateApplicationACLsManager(conf);
     historyManager   = CreateApplicationHistoryManager(conf);
     ahsClientService = CreateApplicationHistoryClientService(historyManager);
     AddService(ahsClientService);
     AddService((Org.Apache.Hadoop.Service.Service)historyManager);
     DefaultMetricsSystem.Initialize("ApplicationHistoryServer");
     JvmMetrics.InitSingleton("ApplicationHistoryServer", null);
     base.ServiceInit(conf);
 }
Ejemplo n.º 28
0
        public virtual void TestNames()
        {
            DefaultMetricsSystem.Initialize("NodeManager");
            NodeManagerMetrics metrics = NodeManagerMetrics.Create();
            Resource           total   = Records.NewRecord <Resource>();

            total.SetMemory(8 * GiB);
            total.SetVirtualCores(16);
            Resource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();

            resource.SetMemory(512);
            //512MiB
            resource.SetVirtualCores(2);
            metrics.AddResource(total);
            for (int i = 10; i-- > 0;)
            {
                // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
                metrics.LaunchedContainer();
                metrics.AllocateContainer(resource);
            }
            metrics.InitingContainer();
            metrics.EndInitingContainer();
            metrics.RunningContainer();
            metrics.EndRunningContainer();
            // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
            metrics.CompletedContainer();
            metrics.ReleaseContainer(resource);
            metrics.FailedContainer();
            metrics.ReleaseContainer(resource);
            metrics.KilledContainer();
            metrics.ReleaseContainer(resource);
            metrics.InitingContainer();
            metrics.RunningContainer();
            NUnit.Framework.Assert.IsTrue(!metrics.containerLaunchDuration.Changed());
            metrics.AddContainerLaunchDuration(1);
            NUnit.Framework.Assert.IsTrue(metrics.containerLaunchDuration.Changed());
            // availableGB is expected to be floored,
            // while allocatedGB is expected to be ceiled.
            // allocatedGB: 3.5GB allocated memory is shown as 4GB
            // availableGB: 4.5GB available memory is shown as 4GB
            CheckMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2);
        }
Ejemplo n.º 29
0
 public virtual void SetUp()
 {
     configuration = new Configuration();
     UserGroupInformation.SetConfiguration(configuration);
     configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true);
     configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
     foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration
                                                                            ))
     {
         configuration.Set(HAUtil.AddSuffix(confKey, Rm1NodeId), Rm1Address);
         configuration.Set(HAUtil.AddSuffix(confKey, Rm2NodeId), Rm2Address);
         configuration.Set(HAUtil.AddSuffix(confKey, Rm3NodeId), Rm3Address);
     }
     // Enable webapp to test web-services also
     configuration.SetBoolean(MockRM.EnableWebapp, true);
     configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
     ClusterMetrics.Destroy();
     QueueMetrics.ClearQueueMetrics();
     DefaultMetricsSystem.Shutdown();
 }
Ejemplo n.º 30
0
        public static void Unregister(ObjectName mbeanName)
        {
            Log.Debug("Unregistering " + mbeanName);
            MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();

            if (mbeanName == null)
            {
                Log.Debug("Stacktrace: ", new Exception());
                return;
            }
            try
            {
                mbs.UnregisterMBean(mbeanName);
            }
            catch (Exception e)
            {
                Log.Warn("Error unregistering " + mbeanName, e);
            }
            DefaultMetricsSystem.RemoveMBeanName(mbeanName);
        }