/// <exception cref="System.Exception"/> private static void PrepareTimelineStore(TimelineStore store, int scale) { for (int i = 1; i <= scale; ++i) { TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.NewInstance(0, i); if (i == 2) { entities.AddEntity(CreateApplicationTimelineEntity(appId, true, false, false)); } else { entities.AddEntity(CreateApplicationTimelineEntity(appId, false, false, false)); } store.Put(entities); for (int j = 1; j <= scale; ++j) { entities = new TimelineEntities(); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j); entities.AddEntity(CreateAppAttemptTimelineEntity(appAttemptId)); store.Put(entities); for (int k = 1; k <= scale; ++k) { entities = new TimelineEntities(); ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k); entities.AddEntity(CreateContainerEntity(containerId)); store.Put(entities); } } } }
public TimelineDataManager(TimelineStore store, TimelineACLsManager timelineACLsManager ) : base(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.TimelineDataManager).FullName ) { this.store = store; this.timelineACLsManager = timelineACLsManager; timelineACLsManager.SetTimelineStore(store); }
public static void PrepareStore() { store = CreateStore(Scale); TimelineEntities entities = new TimelineEntities(); entities.AddEntity(CreateApplicationTimelineEntity(ApplicationId.NewInstance(0, Scale + 1), true, true, false)); entities.AddEntity(CreateApplicationTimelineEntity(ApplicationId.NewInstance(0, Scale + 2), true, false, true)); store.Put(entities); }
public virtual void TestMRTimelineEventHandling() { Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true); MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1); cluster.Init(conf); cluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + cluster.GetApplicationHistoryServer().GetPort()); TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore(); Path inDir = new Path("input"); Path outDir = new Path("output"); RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState( ).GetValue()); TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null , null, null, null, null, null); NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count); TimelineEntity tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents ()[0].GetEventType()); job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir); NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue ()); entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null , null, null); NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count); tEntity = entities.GetEntities()[0]; NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId()); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType()); NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents ()[tEntity.GetEvents().Count - 1].GetEventType()); NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents ()[0].GetEventType()); } finally { if (cluster != null) { cluster.Stop(); } } }
public static void Setup() { Configuration conf = new YarnConfiguration(); TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps ); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore (dataManager, appAclsManager); historyManager.Init(conf); historyManager.Start(); clientService = new ApplicationHistoryClientService(historyManager); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { // init timeline services first timelineStore = CreateTimelineStore(conf); AddIfService(timelineStore); secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf); AddService(secretManagerService); timelineDataManager = CreateTimelineDataManager(conf); AddService(timelineDataManager); // init generic history service afterwards aclsManager = CreateApplicationACLsManager(conf); historyManager = CreateApplicationHistoryManager(conf); ahsClientService = CreateApplicationHistoryClientService(historyManager); AddService(ahsClientService); AddService((Org.Apache.Hadoop.Service.Service)historyManager); DefaultMetricsSystem.Initialize("ApplicationHistoryServer"); JvmMetrics.InitSingleton("ApplicationHistoryServer", null); base.ServiceInit(conf); }
public static void Setup() { YarnConfiguration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.SetBoolean(YarnConfiguration.RmSystemMetricsPublisherEnabled, true); conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore) , typeof(TimelineStore)); conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(MemoryTimelineStateStore ), typeof(TimelineStateStore)); conf.SetInt(YarnConfiguration.RmSystemMetricsPublisherDispatcherPoolSize, 2); timelineServer = new ApplicationHistoryServer(); timelineServer.Init(conf); timelineServer.Start(); store = timelineServer.GetTimelineStore(); metricsPublisher = new SystemMetricsPublisher(); metricsPublisher.Init(conf); metricsPublisher.Start(); }
public static void SetupClass() { Configuration conf = new YarnConfiguration(); TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps ); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); conf.Set(YarnConfiguration.YarnAdminAcl, "foo"); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore (dataManager, appAclsManager); historyManager.Init(conf); historyClientService = new _ApplicationHistoryClientService_98(historyManager); // Do Nothing historyClientService.Init(conf); historyClientService.Start(); }
public static void SetupServer() { conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore) , typeof(TimelineStore)); conf.Set(YarnConfiguration.YarnHttpPolicyKey, "HTTPS_ONLY"); FilePath @base = new FilePath(Basedir); FileUtil.FullyDelete(@base); @base.Mkdirs(); keystoresDir = new FilePath(Basedir).GetAbsolutePath(); sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestTimelineWebServicesWithSSL )); KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.AddResource("ssl-server.xml"); conf.AddResource("ssl-client.xml"); timelineServer = new ApplicationHistoryServer(); timelineServer.Init(conf); timelineServer.Start(); store = timelineServer.GetTimelineStore(); }
public virtual void SetTimelineStore(TimelineStore store) { this.store = store; }